{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 7: Working with Nested Data Structures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Technical Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows: 40949\n",
      "Columns: 16\n",
      "$ video_id                             <str> '2kyS6SvSYSE', '1ZAPwfrtAFY'\n",
      "$ trending_date                        <str> '17.14.11', '17.14.11'\n",
      "$ title                                <str> 'WE WANT TO TALK ABOUT OUR MARRIAGE', 'The Trump Presidency: Last Week Tonight with John Oliver (HBO)'\n",
      "$ channel_title                        <str> 'CaseyNeistat', 'LastWeekTonight'\n",
      "$ category_id                          <i64> 22, 24\n",
      "$ publish_time           <datetime[μs, UTC]> 2017-11-13 17:13:01+00:00, 2017-11-13 07:30:00+00:00\n",
      "$ tags                                 <str> 'SHANtell martin', 'last week tonight trump presidency|last week tonight donald trump|john oliver trump|donald trump'\n",
      "$ views                                <i64> 748374, 2418783\n",
      "$ likes                                <i64> 57527, 97185\n",
      "$ dislikes                             <i64> 2966, 6146\n",
      "$ comment_count                        <i64> 15954, 12703\n",
      "$ thumbnail_link                       <str> 'https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg', 'https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg'\n",
      "$ comments_disabled                   <bool> False, False\n",
      "$ ratings_disabled                    <bool> False, False\n",
      "$ video_error_or_removed              <bool> False, False\n",
      "$ description                          <str> \"SHANTELL'S CHANNEL - https://www.youtube.com/shantellmartin\\\\nCANDICE - https://www.lovebilly.com\\\\n\\\\nfilmed this video in 4k on this -- http://amzn.to/2sTDnRZ\\\\nwith this lens -- http://amzn.to/2rUJOmD\\\\nbig drone - http://tinyurl.com/h4ft3oy\\\\nOTHER GEAR ---  http://amzn.to/2o3GLX5\\\\nSony CAMERA http://amzn.to/2nOBmnv\\\\nOLD CAMERA; http://amzn.to/2o2cQBT\\\\nMAIN LENS; http://amzn.to/2od5gBJ\\\\nBIG SONY CAMERA; http://amzn.to/2nrdJRO\\\\nBIG Canon CAMERA; http://tinyurl.com/jn4q4vz\\\\nBENDY TRIPOD THING; http://tinyurl.com/gw3ylz2\\\\nYOU NEED THIS FOR THE BENDY TRIPOD; http://tinyurl.com/j8mzzua\\\\nWIDE LENS; http://tinyurl.com/jkfcm8t\\\\nMORE EXPENSIVE WIDE LENS; http://tinyurl.com/zrdgtou\\\\nSMALL CAMERA; http://tinyurl.com/hrrzhor\\\\nMICROPHONE; http://tinyurl.com/zefm4jy\\\\nOTHER MICROPHONE; http://tinyurl.com/jxgpj86\\\\nOLD DRONE (cheaper but still great);http://tinyurl.com/zcfmnmd\\\\n\\\\nfollow me; on http://instagram.com/caseyneistat\\\\non https://www.facebook.com/cneistat\\\\non https://twitter.com/CaseyNeistat\\\\n\\\\namazing intro song by https://soundcloud.com/discoteeth\\\\n\\\\nad disclosure.  THIS IS NOT AN AD.  not selling or promoting anything.  but samsung did produce the Shantell Video as a 'GALAXY PROJECT' which is an initiative that enables creators like Shantell and me to make projects we might otherwise not have the opportunity to make.  hope that's clear.  if not ask in the comments and i'll answer any specifics.\", \"One year after the presidential election, John Oliver discusses what we've learned so far and enlists our catheter cowboy to teach Donald Trump what he hasn't.\\\\n\\\\nConnect with Last Week Tonight online...\\\\n\\\\nSubscribe to the Last Week Tonight YouTube channel for more almost news as it almost happens: www.youtube.com/user/LastWeekTonight\\\\n\\\\nFind Last Week Tonight on Facebook like your mom would: http://Facebook.com/LastWeekTonight\\\\n\\\\nFollow us on Twitter for news about jokes and jokes about news: http://Twitter.com/LastWeekTonight\\\\n\\\\nVisit our official site for all that other stuff at once: http://www.hbo.com/lastweektonight\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pl.read_csv(\"../data/us_videos.csv\", try_parse_dates=True)\n",
    "df.glimpse(max_items_per_column=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>video_id</th><th>trending_date</th><th>title</th><th>channel_title</th><th>category_id</th><th>publish_time</th><th>tags</th><th>views</th><th>likes</th><th>dislikes</th><th>comment_count</th><th>thumbnail_link</th><th>comments_disabled</th><th>ratings_disabled</th><th>video_error_or_removed</th><th>description</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>datetime[μs, UTC]</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>bool</td><td>bool</td><td>bool</td><td>str</td></tr></thead><tbody><tr><td>&quot;2kyS6SvSYSE&quot;</td><td>&quot;17.14.11&quot;</td><td>&quot;WE WANT TO TAL…</td><td>&quot;CaseyNeistat&quot;</td><td>22</td><td>2017-11-13 17:13:01 UTC</td><td>&quot;SHANtell marti…</td><td>748374</td><td>57527</td><td>2966</td><td>15954</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;SHANTELL&#x27;S CHA…</td></tr><tr><td>&quot;1ZAPwfrtAFY&quot;</td><td>&quot;17.14.11&quot;</td><td>&quot;The Trump Pres…</td><td>&quot;LastWeekTonigh…</td><td>24</td><td>2017-11-13 07:30:00 UTC</td><td>&quot;last week toni…</td><td>2418783</td><td>97185</td><td>6146</td><td>12703</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;One year after…</td></tr><tr><td>&quot;5qpjK5DgCt4&quot;</td><td>&quot;17.14.11&quot;</td><td>&quot;Racist Superma…</td><td>&quot;Rudy Mancuso&quot;</td><td>23</td><td>2017-11-12 19:05:24 UTC</td><td>&quot;racist superma…</td><td>3191434</td><td>146033</td><td>5339</td><td>8181</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;WATCH MY PREVI…</td></tr><tr><td>&quot;puqaWrEC7tY&quot;</td><td>&quot;17.14.11&quot;</td><td>&quot;Nickelback Lyr…</td><td>&quot;Good Mythical …</td><td>24</td><td>2017-11-13 11:00:04 UTC</td><td>&quot;rhett and link…</td><td>343168</td><td>10172</td><td>666</td><td>2146</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;Today we find …</td></tr><tr><td>&quot;d380meD0W0M&quot;</td><td>&quot;17.14.11&quot;</td><td>&quot;I Dare You: GO…</td><td>&quot;nigahiga&quot;</td><td>24</td><td>2017-11-12 18:01:41 UTC</td><td>&quot;ryan|higa|higa…</td><td>2095731</td><td>132235</td><td>1989</td><td>17518</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;I know it&#x27;s be…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 16)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ video_id  ┆ trending_ ┆ title     ┆ channel_t ┆ … ┆ comments_ ┆ ratings_d ┆ video_err ┆ descript │\n",
       "│ ---       ┆ date      ┆ ---       ┆ itle      ┆   ┆ disabled  ┆ isabled   ┆ or_or_rem ┆ ion      │\n",
       "│ str       ┆ ---       ┆ str       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ oved      ┆ ---      │\n",
       "│           ┆ str       ┆           ┆ str       ┆   ┆ bool      ┆ bool      ┆ ---       ┆ str      │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ bool      ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 2kyS6SvSY ┆ 17.14.11  ┆ WE WANT   ┆ CaseyNeis ┆ … ┆ false     ┆ false     ┆ false     ┆ SHANTELL │\n",
       "│ SE        ┆           ┆ TO TALK   ┆ tat       ┆   ┆           ┆           ┆           ┆ 'S       │\n",
       "│           ┆           ┆ ABOUT OUR ┆           ┆   ┆           ┆           ┆           ┆ CHANNEL  │\n",
       "│           ┆           ┆ MARRIA…   ┆           ┆   ┆           ┆           ┆           ┆ - https: │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ //www…   │\n",
       "│ 1ZAPwfrtA ┆ 17.14.11  ┆ The Trump ┆ LastWeekT ┆ … ┆ false     ┆ false     ┆ false     ┆ One year │\n",
       "│ FY        ┆           ┆ Presidenc ┆ onight    ┆   ┆           ┆           ┆           ┆ after    │\n",
       "│           ┆           ┆ y: Last   ┆           ┆   ┆           ┆           ┆           ┆ the pres │\n",
       "│           ┆           ┆ Week …    ┆           ┆   ┆           ┆           ┆           ┆ idential │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ …        │\n",
       "│ 5qpjK5DgC ┆ 17.14.11  ┆ Racist    ┆ Rudy      ┆ … ┆ false     ┆ false     ┆ false     ┆ WATCH MY │\n",
       "│ t4        ┆           ┆ Superman  ┆ Mancuso   ┆   ┆           ┆           ┆           ┆ PREVIOUS │\n",
       "│           ┆           ┆ | Rudy    ┆           ┆   ┆           ┆           ┆           ┆ VIDEO ▶  │\n",
       "│           ┆           ┆ Mancuso,  ┆           ┆   ┆           ┆           ┆           ┆ \\n\\nSU…  │\n",
       "│           ┆           ┆ …         ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ puqaWrEC7 ┆ 17.14.11  ┆ Nickelbac ┆ Good      ┆ … ┆ false     ┆ false     ┆ false     ┆ Today we │\n",
       "│ tY        ┆           ┆ k Lyrics: ┆ Mythical  ┆   ┆           ┆           ┆           ┆ find out │\n",
       "│           ┆           ┆ Real or   ┆ Morning   ┆   ┆           ┆           ┆           ┆ if Link  │\n",
       "│           ┆           ┆ Fake?     ┆           ┆   ┆           ┆           ┆           ┆ is a N…  │\n",
       "│ d380meD0W ┆ 17.14.11  ┆ I Dare    ┆ nigahiga  ┆ … ┆ false     ┆ false     ┆ false     ┆ I know   │\n",
       "│ 0M        ┆           ┆ You:      ┆           ┆   ┆           ┆           ┆           ┆ it's     │\n",
       "│           ┆           ┆ GOING     ┆           ┆   ┆           ┆           ┆           ┆ been a   │\n",
       "│           ┆           ┆ BALD!?    ┆           ┆   ┆           ┆           ┆           ┆ while    │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ since w… │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.with_columns(pl.col(\"trending_date\").str.to_date(format=\"%y.%d.%m\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Date"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('trending_date').dtypes[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating lists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>tags</th><th>tags in list</th></tr><tr><td>str</td><td>list[str]</td></tr></thead><tbody><tr><td>&quot;SHANtell marti…</td><td>[&quot;SHANtell martin&quot;]</td></tr><tr><td>&quot;last week toni…</td><td>[&quot;last week tonight trump presidency&quot;, &quot;last week tonight donald trump&quot;, … &quot;donald trump&quot;]</td></tr><tr><td>&quot;racist superma…</td><td>[&quot;racist superman&quot;, &quot;rudy&quot;, … &quot; Lele Pons&quot;]</td></tr><tr><td>&quot;rhett and link…</td><td>[&quot;rhett and link&quot;, &quot;gmm&quot;, … &quot;challenge&quot;]</td></tr><tr><td>&quot;ryan|higa|higa…</td><td>[&quot;ryan&quot;, &quot;higa&quot;, … &quot;fail&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ tags                              ┆ tags in list                      │\n",
       "│ ---                               ┆ ---                               │\n",
       "│ str                               ┆ list[str]                         │\n",
       "╞═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ SHANtell martin                   ┆ [\"SHANtell martin\"]               │\n",
       "│ last week tonight trump presiden… ┆ [\"last week tonight trump presid… │\n",
       "│ racist superman|rudy|mancuso|kin… ┆ [\"racist superman\", \"rudy\", … \" … │\n",
       "│ rhett and link|gmm|good mythical… ┆ [\"rhett and link\", \"gmm\", … \"cha… │\n",
       "│ ryan|higa|higatv|nigahiga|i dare… ┆ [\"ryan\", \"higa\", … \"fail\"]        │\n",
       "└───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(\n",
    "    'tags',\n",
    "    pl.col('tags').str.split('|').alias('tags in list')\n",
    ").head()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>video_id</th></tr><tr><td>date</td><td>list[str]</td></tr></thead><tbody><tr><td>2018-06-14</td><td>[&quot;-QPdRfqTnt4&quot;, &quot;gPHVLxm8U-0&quot;, … &quot;ooyjaVdt-jA&quot;]</td></tr><tr><td>2018-06-13</td><td>[&quot;FchkqXEg0qs&quot;, &quot;uHRwMmwbFnA&quot;, … &quot;Q5KmA3Xbmqo&quot;]</td></tr><tr><td>2018-06-12</td><td>[&quot;PPWDwBrUNyY&quot;, &quot;rAH8qm5oQHg&quot;, … &quot;6S9c5nnDd_s&quot;]</td></tr><tr><td>2018-06-11</td><td>[&quot;0bXCbVGb04A&quot;, &quot;L4pkD78oKSo&quot;, … &quot;r-3iathMo7o&quot;]</td></tr><tr><td>2018-06-10</td><td>[&quot;L4pkD78oKSo&quot;, &quot;ZFwylDNpgFc&quot;, … &quot;r-3iathMo7o&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────┬───────────────────────────────────┐\n",
       "│ trending_date ┆ video_id                          │\n",
       "│ ---           ┆ ---                               │\n",
       "│ date          ┆ list[str]                         │\n",
       "╞═══════════════╪═══════════════════════════════════╡\n",
       "│ 2018-06-14    ┆ [\"-QPdRfqTnt4\", \"gPHVLxm8U-0\", …… │\n",
       "│ 2018-06-13    ┆ [\"FchkqXEg0qs\", \"uHRwMmwbFnA\", …… │\n",
       "│ 2018-06-12    ┆ [\"PPWDwBrUNyY\", \"rAH8qm5oQHg\", …… │\n",
       "│ 2018-06-11    ┆ [\"0bXCbVGb04A\", \"L4pkD78oKSo\", …… │\n",
       "│ 2018-06-10    ┆ [\"L4pkD78oKSo\", \"ZFwylDNpgFc\", …… │\n",
       "└───────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .group_by('trending_date')\n",
    "    .agg(pl.col('video_id'))\n",
    "    .sort('trending_date', descending=True)\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>engagement</th></tr><tr><td>list[i64]</td></tr></thead><tbody><tr><td>[748374, 57527, … 15954]</td></tr><tr><td>[2418783, 97185, … 12703]</td></tr><tr><td>[3191434, 146033, … 8181]</td></tr><tr><td>[343168, 10172, … 2146]</td></tr><tr><td>[2095731, 132235, … 17518]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 1)\n",
       "┌────────────────────────────┐\n",
       "│ engagement                 │\n",
       "│ ---                        │\n",
       "│ list[i64]                  │\n",
       "╞════════════════════════════╡\n",
       "│ [748374, 57527, … 15954]   │\n",
       "│ [2418783, 97185, … 12703]  │\n",
       "│ [3191434, 146033, … 8181]  │\n",
       "│ [343168, 10172, … 2146]    │\n",
       "│ [2095731, 132235, … 17518] │\n",
       "└────────────────────────────┘"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(\n",
    "    pl.concat_list(\n",
    "        pl.col('views'),\n",
    "        pl.col('likes'),\n",
    "        pl.col('dislikes'),\n",
    "        pl.col('comment_count')\n",
    "    ).alias('engagement')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pl.DataFrame({\n",
    "    'nested_list': [\n",
    "        [\n",
    "            [1,2,3], [4,5,6], \n",
    "            [7,8,9], [10,11,12]\n",
    "        ], \n",
    "        [\n",
    "            [1,2,3], [4,5,6], \n",
    "            [7,8,9], [10,11,12]\n",
    "        ]\n",
    "    ]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>nested_list</th></tr><tr><td>list[list[i64]]</td></tr></thead><tbody><tr><td>[[1, 2, 3], [4, 5, 6], … [10, 11, 12]]</td></tr><tr><td>[[1, 2, 3], [4, 5, 6], … [10, 11, 12]]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 1)\n",
       "┌───────────────────────────────────┐\n",
       "│ nested_list                       │\n",
       "│ ---                               │\n",
       "│ list[list[i64]]                   │\n",
       "╞═══════════════════════════════════╡\n",
       "│ [[1, 2, 3], [4, 5, 6], … [10, 11… │\n",
       "│ [[1, 2, 3], [4, 5, 6], … [10, 11… │\n",
       "└───────────────────────────────────┘"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pl.DataFrame({\n",
    "    'nested_list': [\n",
    "        [\n",
    "            ['a',2,3], [4,5,6], \n",
    "            [7,8,9], [10,11,12]\n",
    "        ], \n",
    "        [\n",
    "            [1,2,3], [4,5,6], \n",
    "            [7,8,9], [10,11,12]\n",
    "        ]\n",
    "    ]\n",
    "}, strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>nested_list</th></tr><tr><td>list[list[str]]</td></tr></thead><tbody><tr><td>[[&quot;a&quot;, &quot;2&quot;, &quot;3&quot;], [&quot;4&quot;, &quot;5&quot;, &quot;6&quot;], … [&quot;10&quot;, &quot;11&quot;, &quot;12&quot;]]</td></tr><tr><td>[[&quot;1&quot;, &quot;2&quot;, &quot;3&quot;], [&quot;4&quot;, &quot;5&quot;, &quot;6&quot;], … [&quot;10&quot;, &quot;11&quot;, &quot;12&quot;]]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2, 1)\n",
       "┌───────────────────────────────────┐\n",
       "│ nested_list                       │\n",
       "│ ---                               │\n",
       "│ list[list[str]]                   │\n",
       "╞═══════════════════════════════════╡\n",
       "│ [[\"a\", \"2\", \"3\"], [\"4\", \"5\", \"6\"… │\n",
       "│ [[\"1\", \"2\", \"3\"], [\"4\", \"5\", \"6\"… │\n",
       "└───────────────────────────────────┘"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aggregating elements in lists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>video_id</th><th>trending_date</th><th>title</th><th>channel_title</th><th>category_id</th><th>publish_time</th><th>tags</th><th>views</th><th>likes</th><th>dislikes</th><th>comment_count</th><th>thumbnail_link</th><th>comments_disabled</th><th>ratings_disabled</th><th>video_error_or_removed</th><th>description</th></tr><tr><td>str</td><td>date</td><td>str</td><td>str</td><td>i64</td><td>datetime[μs, UTC]</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>bool</td><td>bool</td><td>bool</td><td>str</td></tr></thead><tbody><tr><td>&quot;2kyS6SvSYSE&quot;</td><td>2017-11-14</td><td>&quot;WE WANT TO TAL…</td><td>&quot;CaseyNeistat&quot;</td><td>22</td><td>2017-11-13 17:13:01 UTC</td><td>&quot;SHANtell marti…</td><td>748374</td><td>57527</td><td>2966</td><td>15954</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;SHANTELL&#x27;S CHA…</td></tr><tr><td>&quot;1ZAPwfrtAFY&quot;</td><td>2017-11-14</td><td>&quot;The Trump Pres…</td><td>&quot;LastWeekTonigh…</td><td>24</td><td>2017-11-13 07:30:00 UTC</td><td>&quot;last week toni…</td><td>2418783</td><td>97185</td><td>6146</td><td>12703</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;One year after…</td></tr><tr><td>&quot;5qpjK5DgCt4&quot;</td><td>2017-11-14</td><td>&quot;Racist Superma…</td><td>&quot;Rudy Mancuso&quot;</td><td>23</td><td>2017-11-12 19:05:24 UTC</td><td>&quot;racist superma…</td><td>3191434</td><td>146033</td><td>5339</td><td>8181</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;WATCH MY PREVI…</td></tr><tr><td>&quot;puqaWrEC7tY&quot;</td><td>2017-11-14</td><td>&quot;Nickelback Lyr…</td><td>&quot;Good Mythical …</td><td>24</td><td>2017-11-13 11:00:04 UTC</td><td>&quot;rhett and link…</td><td>343168</td><td>10172</td><td>666</td><td>2146</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;Today we find …</td></tr><tr><td>&quot;d380meD0W0M&quot;</td><td>2017-11-14</td><td>&quot;I Dare You: GO…</td><td>&quot;nigahiga&quot;</td><td>24</td><td>2017-11-12 18:01:41 UTC</td><td>&quot;ryan|higa|higa…</td><td>2095731</td><td>132235</td><td>1989</td><td>17518</td><td>&quot;https://i.ytim…</td><td>false</td><td>false</td><td>false</td><td>&quot;I know it&#x27;s be…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 16)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ video_id  ┆ trending_ ┆ title     ┆ channel_t ┆ … ┆ comments_ ┆ ratings_d ┆ video_err ┆ descript │\n",
       "│ ---       ┆ date      ┆ ---       ┆ itle      ┆   ┆ disabled  ┆ isabled   ┆ or_or_rem ┆ ion      │\n",
       "│ str       ┆ ---       ┆ str       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ oved      ┆ ---      │\n",
       "│           ┆ date      ┆           ┆ str       ┆   ┆ bool      ┆ bool      ┆ ---       ┆ str      │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ bool      ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 2kyS6SvSY ┆ 2017-11-1 ┆ WE WANT   ┆ CaseyNeis ┆ … ┆ false     ┆ false     ┆ false     ┆ SHANTELL │\n",
       "│ SE        ┆ 4         ┆ TO TALK   ┆ tat       ┆   ┆           ┆           ┆           ┆ 'S       │\n",
       "│           ┆           ┆ ABOUT OUR ┆           ┆   ┆           ┆           ┆           ┆ CHANNEL  │\n",
       "│           ┆           ┆ MARRIA…   ┆           ┆   ┆           ┆           ┆           ┆ - https: │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ //www…   │\n",
       "│ 1ZAPwfrtA ┆ 2017-11-1 ┆ The Trump ┆ LastWeekT ┆ … ┆ false     ┆ false     ┆ false     ┆ One year │\n",
       "│ FY        ┆ 4         ┆ Presidenc ┆ onight    ┆   ┆           ┆           ┆           ┆ after    │\n",
       "│           ┆           ┆ y: Last   ┆           ┆   ┆           ┆           ┆           ┆ the pres │\n",
       "│           ┆           ┆ Week …    ┆           ┆   ┆           ┆           ┆           ┆ idential │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ …        │\n",
       "│ 5qpjK5DgC ┆ 2017-11-1 ┆ Racist    ┆ Rudy      ┆ … ┆ false     ┆ false     ┆ false     ┆ WATCH MY │\n",
       "│ t4        ┆ 4         ┆ Superman  ┆ Mancuso   ┆   ┆           ┆           ┆           ┆ PREVIOUS │\n",
       "│           ┆           ┆ | Rudy    ┆           ┆   ┆           ┆           ┆           ┆ VIDEO ▶  │\n",
       "│           ┆           ┆ Mancuso,  ┆           ┆   ┆           ┆           ┆           ┆ \\n\\nSU…  │\n",
       "│           ┆           ┆ …         ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ puqaWrEC7 ┆ 2017-11-1 ┆ Nickelbac ┆ Good      ┆ … ┆ false     ┆ false     ┆ false     ┆ Today we │\n",
       "│ tY        ┆ 4         ┆ k Lyrics: ┆ Mythical  ┆   ┆           ┆           ┆           ┆ find out │\n",
       "│           ┆           ┆ Real or   ┆ Morning   ┆   ┆           ┆           ┆           ┆ if Link  │\n",
       "│           ┆           ┆ Fake?     ┆           ┆   ┆           ┆           ┆           ┆ is a N…  │\n",
       "│ d380meD0W ┆ 2017-11-1 ┆ I Dare    ┆ nigahiga  ┆ … ┆ false     ┆ false     ┆ false     ┆ I know   │\n",
       "│ 0M        ┆ 4         ┆ You:      ┆           ┆   ┆           ┆           ┆           ┆ it's     │\n",
       "│           ┆           ┆ GOING     ┆           ┆   ┆           ┆           ┆           ┆ been a   │\n",
       "│           ┆           ┆ BALD!?    ┆           ┆   ┆           ┆           ┆           ┆ while    │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ since w… │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = (\n",
    "    pl.read_csv('../data/us_videos.csv', try_parse_dates=True)\n",
    "    .with_columns(\n",
    "        pl.col('trending_date').str.to_date(format='%y.%d.%m')\n",
    "    )\n",
    ")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>likes</th><th>dislikes</th><th>comment_count</th></tr><tr><td>date</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>2018-06-14</td><td>[4427381, 5829270, … 10306119]</td><td>[96391, 87323, … 357079]</td><td>[5508, 3668, … 212976]</td><td>[12726, 11933, … 144795]</td></tr><tr><td>2018-06-13</td><td>[3238183, 470844, … 7839668]</td><td>[61841, 13922, … 352352]</td><td>[3708, 402, … 5871]</td><td>[0, 4843, … 46624]</td></tr><tr><td>2018-06-12</td><td>[3483553, 6173038, … 13619534]</td><td>[23725, 90478, … 347100]</td><td>[3145, 3877, … 6923]</td><td>[462, 7726, … 19977]</td></tr><tr><td>2018-06-11</td><td>[2341772, 846887, … 6995168]</td><td>[140374, 5758, … 143678]</td><td>[2951, 850, … 10925]</td><td>[33760, 1539, … 17444]</td></tr><tr><td>2018-06-10</td><td>[594004, 1504564, … 6980540]</td><td>[4470, 32430, … 143452]</td><td>[520, 4316, … 10911]</td><td>[1195, 6105, … 17429]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌───────────────┬───────────────────────┬──────────────────┬────────────────┬──────────────────────┐\n",
       "│ trending_date ┆ views                 ┆ likes            ┆ dislikes       ┆ comment_count        │\n",
       "│ ---           ┆ ---                   ┆ ---              ┆ ---            ┆ ---                  │\n",
       "│ date          ┆ list[i64]             ┆ list[i64]        ┆ list[i64]      ┆ list[i64]            │\n",
       "╞═══════════════╪═══════════════════════╪══════════════════╪════════════════╪══════════════════════╡\n",
       "│ 2018-06-14    ┆ [4427381, 5829270, …  ┆ [96391, 87323, … ┆ [5508, 3668, … ┆ [12726, 11933, …     │\n",
       "│               ┆ 10306119]             ┆ 357079]          ┆ 212976]        ┆ 144795]              │\n",
       "│ 2018-06-13    ┆ [3238183, 470844, …   ┆ [61841, 13922, … ┆ [3708, 402, …  ┆ [0, 4843, … 46624]   │\n",
       "│               ┆ 7839668]              ┆ 352352]          ┆ 5871]          ┆                      │\n",
       "│ 2018-06-12    ┆ [3483553, 6173038, …  ┆ [23725, 90478, … ┆ [3145, 3877, … ┆ [462, 7726, … 19977] │\n",
       "│               ┆ 13619534]             ┆ 347100]          ┆ 6923]          ┆                      │\n",
       "│ 2018-06-11    ┆ [2341772, 846887, …   ┆ [140374, 5758, … ┆ [2951, 850, …  ┆ [33760, 1539, …      │\n",
       "│               ┆ 6995168]              ┆ 143678]          ┆ 10925]         ┆ 17444]               │\n",
       "│ 2018-06-10    ┆ [594004, 1504564, …   ┆ [4470, 32430, …  ┆ [520, 4316, …  ┆ [1195, 6105, …       │\n",
       "│               ┆ 6980540]              ┆ 143452]          ┆ 10911]         ┆ 17429]               │\n",
       "└───────────────┴───────────────────────┴──────────────────┴────────────────┴──────────────────────┘"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_df = (\n",
    "    df\n",
    "    .group_by('trending_date')\n",
    "    .agg(\n",
    "        'views',\n",
    "        'likes',\n",
    "        'dislikes',\n",
    "        'comment_count'\n",
    "    )\n",
    "    .sort('trending_date', descending=True)\n",
    ")\n",
    "agg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views_min</th><th>likes_max</th><th>dislikes_mean</th><th>comment_sum</th></tr><tr><td>date</td><td>i64</td><td>i64</td><td>f64</td><td>i64</td></tr></thead><tbody><tr><td>2018-06-14</td><td>189038</td><td>2032463</td><td>8472.94</td><td>3289433</td></tr><tr><td>2018-06-13</td><td>175754</td><td>2021395</td><td>8541.23</td><td>3379632</td></tr><tr><td>2018-06-12</td><td>161782</td><td>2004753</td><td>8353.685</td><td>3352328</td></tr><tr><td>2018-06-11</td><td>136643</td><td>1981942</td><td>8478.095</td><td>3320709</td></tr><tr><td>2018-06-10</td><td>116841</td><td>1967904</td><td>8405.22</td><td>3351105</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌───────────────┬───────────┬───────────┬───────────────┬─────────────┐\n",
       "│ trending_date ┆ views_min ┆ likes_max ┆ dislikes_mean ┆ comment_sum │\n",
       "│ ---           ┆ ---       ┆ ---       ┆ ---           ┆ ---         │\n",
       "│ date          ┆ i64       ┆ i64       ┆ f64           ┆ i64         │\n",
       "╞═══════════════╪═══════════╪═══════════╪═══════════════╪═════════════╡\n",
       "│ 2018-06-14    ┆ 189038    ┆ 2032463   ┆ 8472.94       ┆ 3289433     │\n",
       "│ 2018-06-13    ┆ 175754    ┆ 2021395   ┆ 8541.23       ┆ 3379632     │\n",
       "│ 2018-06-12    ┆ 161782    ┆ 2004753   ┆ 8353.685      ┆ 3352328     │\n",
       "│ 2018-06-11    ┆ 136643    ┆ 1981942   ┆ 8478.095      ┆ 3320709     │\n",
       "│ 2018-06-10    ┆ 116841    ┆ 1967904   ┆ 8405.22       ┆ 3351105     │\n",
       "└───────────────┴───────────┴───────────┴───────────────┴─────────────┘"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    agg_df\n",
    "    .select(\n",
    "        'trending_date',\n",
    "        pl.col('views').list.min().alias('views_min'),\n",
    "        pl.col('likes').list.max().alias('likes_max'),\n",
    "        pl.col('dislikes').list.mean().alias('dislikes_mean'),\n",
    "        pl.col('comment_count').list.sum().alias('comment_sum'),\n",
    "    )\n",
    "    .sort('trending_date', descending=True)\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>channel_title</th></tr><tr><td>date</td><td>str</td></tr></thead><tbody><tr><td>2018-06-14</td><td>&quot;Disney Movie T…</td></tr><tr><td>2018-06-13</td><td>&quot;Nintendo:GameX…</td></tr><tr><td>2018-06-12</td><td>&quot;gameslice:Clas…</td></tr><tr><td>2018-06-11</td><td>&quot;The Game Theor…</td></tr><tr><td>2018-06-10</td><td>&quot;NBC Sports:Ant…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────┬───────────────────────────────────┐\n",
       "│ trending_date ┆ channel_title                     │\n",
       "│ ---           ┆ ---                               │\n",
       "│ date          ┆ str                               │\n",
       "╞═══════════════╪═══════════════════════════════════╡\n",
       "│ 2018-06-14    ┆ Disney Movie Trailers:America's … │\n",
       "│ 2018-06-13    ┆ Nintendo:GameXplain:gameslice:Yo… │\n",
       "│ 2018-06-12    ┆ gameslice:Clash of Clans:FamilyF… │\n",
       "│ 2018-06-11    ┆ The Game Theorists:NBC Sports:An… │\n",
       "│ 2018-06-10    ┆ NBC Sports:Anthem Game:Universal… │\n",
       "└───────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .group_by('trending_date')\n",
    "    .agg(pl.col('channel_title'))\n",
    "    .with_columns(\n",
    "        pl.col('channel_title').list.join(':')\n",
    "    )\n",
    "    .sort('trending_date', descending=True)\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>item_cnt</th></tr><tr><td>date</td><td>u32</td></tr></thead><tbody><tr><td>2018-03-09</td><td>200</td></tr><tr><td>2018-02-22</td><td>199</td></tr><tr><td>2018-02-25</td><td>200</td></tr><tr><td>2018-02-28</td><td>199</td></tr><tr><td>2018-03-03</td><td>199</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────┬──────────┐\n",
       "│ trending_date ┆ item_cnt │\n",
       "│ ---           ┆ ---      │\n",
       "│ date          ┆ u32      │\n",
       "╞═══════════════╪══════════╡\n",
       "│ 2018-03-09    ┆ 200      │\n",
       "│ 2018-02-22    ┆ 199      │\n",
       "│ 2018-02-25    ┆ 200      │\n",
       "│ 2018-02-28    ┆ 199      │\n",
       "│ 2018-03-03    ┆ 199      │\n",
       "└───────────────┴──────────┘"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    agg_df\n",
    "    .select(\n",
    "        'trending_date',\n",
    "        pl.col('views').list.len().alias('item_cnt')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing and selecting elements in lists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "trending_dates_by_channel = (\n",
    "    df\n",
    "    .group_by('channel_title')\n",
    "    .agg('trending_date')\n",
    "    .with_columns(pl.col('trending_date').list.sort())\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th></tr><tr><td>str</td><td>list[date]</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────────────────┬───────────────────────────────────┐\n",
       "│ channel_title   ┆ trending_date                     │\n",
       "│ ---             ┆ ---                               │\n",
       "│ str             ┆ list[date]                        │\n",
       "╞═════════════════╪═══════════════════════════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, … 2018-… │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, … 2017-… │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, … 2018-… │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, … 2018-… │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, … 2018-… │\n",
       "└─────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th><th>first_trending_date</th><th>last_trending_date</th></tr><tr><td>str</td><td>list[date]</td><td>date</td><td>date</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>2018-01-24</td><td>2018-02-12</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>2017-12-02</td><td>2017-12-07</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>2017-12-29</td><td>2018-03-03</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>2018-04-19</td><td>2018-05-14</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>2017-11-28</td><td>2018-01-05</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌─────────────────┬───────────────────────────────────┬─────────────────────┬────────────────────┐\n",
       "│ channel_title   ┆ trending_date                     ┆ first_trending_date ┆ last_trending_date │\n",
       "│ ---             ┆ ---                               ┆ ---                 ┆ ---                │\n",
       "│ str             ┆ list[date]                        ┆ date                ┆ date               │\n",
       "╞═════════════════╪═══════════════════════════════════╪═════════════════════╪════════════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, … 2018-… ┆ 2018-01-24          ┆ 2018-02-12         │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, … 2017-… ┆ 2017-12-02          ┆ 2017-12-07         │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, … 2018-… ┆ 2017-12-29          ┆ 2018-03-03         │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, … 2018-… ┆ 2018-04-19          ┆ 2018-05-14         │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, … 2018-… ┆ 2017-11-28          ┆ 2018-01-05         │\n",
       "└─────────────────┴───────────────────────────────────┴─────────────────────┴────────────────────┘"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col('trending_date').list.first().alias('first_trending_date'),\n",
    "    pl.col('trending_date').list.last().alias('last_trending_date')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th><th>8th_element</th></tr><tr><td>str</td><td>list[date]</td><td>date</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>2018-01-25</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>2017-12-03</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>2017-12-30</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>2018-04-20</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>2017-11-29</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────────────┬───────────────────────────────────┬─────────────┐\n",
       "│ channel_title   ┆ trending_date                     ┆ 8th_element │\n",
       "│ ---             ┆ ---                               ┆ ---         │\n",
       "│ str             ┆ list[date]                        ┆ date        │\n",
       "╞═════════════════╪═══════════════════════════════════╪═════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, … 2018-… ┆ 2018-01-25  │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, … 2017-… ┆ 2017-12-03  │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, … 2018-… ┆ 2017-12-30  │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, … 2018-… ┆ 2018-04-20  │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, … 2018-… ┆ 2017-11-29  │\n",
       "└─────────────────┴───────────────────────────────────┴─────────────┘"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col('trending_date').list.get(7, null_on_oob=True).alias('8th_element')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th><th>first_5</th><th>last_10</th></tr><tr><td>str</td><td>list[date]</td><td>list[date]</td><td>list[date]</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>[2018-01-24, 2018-01-25, … 2018-01-28]</td><td>[2018-01-25, 2018-01-26, … 2018-02-12]</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>[2017-12-02, 2017-12-03, … 2017-12-06]</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>[2017-12-29, 2017-12-30, … 2018-01-02]</td><td>[2018-02-06, 2018-02-07, … 2018-03-03]</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>[2018-04-19, 2018-04-20, … 2018-04-23]</td><td>[2018-04-26, 2018-04-27, … 2018-05-14]</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>[2017-11-28, 2017-11-29, … 2017-12-07]</td><td>[2017-12-27, 2017-12-28, … 2018-01-05]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌─────────────────┬──────────────────────────┬──────────────────────────┬──────────────────────────┐\n",
       "│ channel_title   ┆ trending_date            ┆ first_5                  ┆ last_10                  │\n",
       "│ ---             ┆ ---                      ┆ ---                      ┆ ---                      │\n",
       "│ str             ┆ list[date]               ┆ list[date]               ┆ list[date]               │\n",
       "╞═════════════════╪══════════════════════════╪══════════════════════════╪══════════════════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, ┆ [2018-01-24, 2018-01-25, ┆ [2018-01-25, 2018-01-26, │\n",
       "│                 ┆ … 2018-…                 ┆ … 2018-…                 ┆ … 2018-…                 │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, ┆ [2017-12-02, 2017-12-03, ┆ [2017-12-02, 2017-12-03, │\n",
       "│                 ┆ … 2017-…                 ┆ … 2017-…                 ┆ … 2017-…                 │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, ┆ [2017-12-29, 2017-12-30, ┆ [2018-02-06, 2018-02-07, │\n",
       "│                 ┆ … 2018-…                 ┆ … 2018-…                 ┆ … 2018-…                 │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, ┆ [2018-04-19, 2018-04-20, ┆ [2018-04-26, 2018-04-27, │\n",
       "│                 ┆ … 2018-…                 ┆ … 2018-…                 ┆ … 2018-…                 │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, ┆ [2017-11-28, 2017-11-29, ┆ [2017-12-27, 2017-12-28, │\n",
       "│                 ┆ … 2018-…                 ┆ … 2017-…                 ┆ … 2018-…                 │\n",
       "└─────────────────┴──────────────────────────┴──────────────────────────┴──────────────────────────┘"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col('trending_date').list.head().alias('first_5'),\n",
    "    pl.col('trending_date').list.tail(10).alias('last_10')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th><th>3_most_recent_dates</th></tr><tr><td>str</td><td>list[date]</td><td>list[date]</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>[2018-02-12, 2018-02-11, 2018-02-10]</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>[2017-12-07, 2017-12-06, 2017-12-05]</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>[2018-03-03, 2018-03-02, 2018-02-13]</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>[2018-05-14, 2018-05-04, 2018-05-03]</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>[2018-01-05, 2018-01-04, 2018-01-03]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────────────┬───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ channel_title   ┆ trending_date                     ┆ 3_most_recent_dates               │\n",
       "│ ---             ┆ ---                               ┆ ---                               │\n",
       "│ str             ┆ list[date]                        ┆ list[date]                        │\n",
       "╞═════════════════╪═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, … 2018-… ┆ [2018-02-12, 2018-02-11, 2018-02… │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, … 2017-… ┆ [2017-12-07, 2017-12-06, 2017-12… │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, … 2018-… ┆ [2018-03-03, 2018-03-02, 2018-02… │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, … 2018-… ┆ [2018-05-14, 2018-05-04, 2018-05… │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, … 2018-… ┆ [2018-01-05, 2018-01-04, 2018-01… │\n",
       "└─────────────────┴───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col('trending_date')\n",
    "    .list.sort(descending=True)\n",
    "    .list.head(3)\n",
    "    .alias('3_most_recent_dates')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>trending_date</th><th>3_most_recent_dates</th></tr><tr><td>str</td><td>list[date]</td><td>list[date]</td></tr></thead><tbody><tr><td>&quot;Drew Lynch&quot;</td><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>[2018-02-12, 2018-02-11, 2018-02-10]</td></tr><tr><td>&quot;FIFATV&quot;</td><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>[2017-12-07, 2017-12-06, 2017-12-05]</td></tr><tr><td>&quot;SmarterEveryDa…</td><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>[2018-03-03, 2018-03-02, 2018-02-13]</td></tr><tr><td>&quot;GingerPale&quot;</td><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>[2018-05-14, 2018-05-04, 2018-05-03]</td></tr><tr><td>&quot;Linkin Park&quot;</td><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>[2018-01-05, 2018-01-04, 2018-01-03]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌─────────────────┬───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ channel_title   ┆ trending_date                     ┆ 3_most_recent_dates               │\n",
       "│ ---             ┆ ---                               ┆ ---                               │\n",
       "│ str             ┆ list[date]                        ┆ list[date]                        │\n",
       "╞═════════════════╪═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ Drew Lynch      ┆ [2018-01-24, 2018-01-25, … 2018-… ┆ [2018-02-12, 2018-02-11, 2018-02… │\n",
       "│ FIFATV          ┆ [2017-12-02, 2017-12-03, … 2017-… ┆ [2017-12-07, 2017-12-06, 2017-12… │\n",
       "│ SmarterEveryDay ┆ [2017-12-29, 2017-12-30, … 2018-… ┆ [2018-03-03, 2018-03-02, 2018-02… │\n",
       "│ GingerPale      ┆ [2018-04-19, 2018-04-20, … 2018-… ┆ [2018-05-14, 2018-05-04, 2018-05… │\n",
       "│ Linkin Park     ┆ [2017-11-28, 2017-11-29, … 2018-… ┆ [2018-01-05, 2018-01-04, 2018-01… │\n",
       "└─────────────────┴───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col('trending_date')\n",
    "    .list.sort(descending=True)\n",
    "    .list.head(3)\n",
    "    .alias('3_most_recent_dates')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>first_2_dates</th><th>3rd_date_to_last</th><th>from_8th_date_to_end</th></tr><tr><td>list[date]</td><td>list[date]</td><td>list[date]</td><td>list[date]</td></tr></thead><tbody><tr><td>[2018-03-01, 2018-03-02, … 2018-06-14]</td><td>[2018-03-01, 2018-03-02]</td><td>[2018-06-12]</td><td>[2018-03-08, 2018-03-09, … 2018-06-14]</td></tr><tr><td>[2018-05-15, 2018-05-15]</td><td>[2018-05-15, 2018-05-15]</td><td>[]</td><td>[]</td></tr><tr><td>[2017-11-20, 2017-11-21, … 2018-03-08]</td><td>[2017-11-20, 2017-11-21]</td><td>[2018-03-06]</td><td>[2017-12-01, 2017-12-02, … 2018-03-08]</td></tr><tr><td>[2017-11-14, 2017-11-15, … 2017-11-19]</td><td>[2017-11-14, 2017-11-15]</td><td>[2017-11-17]</td><td>[]</td></tr><tr><td>[2017-11-18, 2017-11-19, … 2018-05-08]</td><td>[2017-11-18, 2017-11-19]</td><td>[2018-05-06]</td><td>[2017-11-25, 2017-11-26, … 2018-05-08]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌──────────────────────┬────────────────────────────┬──────────────────┬───────────────────────────┐\n",
       "│ trending_date        ┆ first_2_dates              ┆ 3rd_date_to_last ┆ from_8th_date_to_end      │\n",
       "│ ---                  ┆ ---                        ┆ ---              ┆ ---                       │\n",
       "│ list[date]           ┆ list[date]                 ┆ list[date]       ┆ list[date]                │\n",
       "╞══════════════════════╪════════════════════════════╪══════════════════╪═══════════════════════════╡\n",
       "│ [2018-03-01,         ┆ [2018-03-01, 2018-03-02]   ┆ [2018-06-12]     ┆ [2018-03-08, 2018-03-09,  │\n",
       "│ 2018-03-02, … 2018-… ┆                            ┆                  ┆ … 2018-…                  │\n",
       "│ [2018-05-15,         ┆ [2018-05-15, 2018-05-15]   ┆ []               ┆ []                        │\n",
       "│ 2018-05-15]          ┆                            ┆                  ┆                           │\n",
       "│ [2017-11-20,         ┆ [2017-11-20, 2017-11-21]   ┆ [2018-03-06]     ┆ [2017-12-01, 2017-12-02,  │\n",
       "│ 2017-11-21, … 2018-… ┆                            ┆                  ┆ … 2018-…                  │\n",
       "│ [2017-11-14,         ┆ [2017-11-14, 2017-11-15]   ┆ [2017-11-17]     ┆ []                        │\n",
       "│ 2017-11-15, … 2017-… ┆                            ┆                  ┆                           │\n",
       "│ [2017-11-18,         ┆ [2017-11-18, 2017-11-19]   ┆ [2018-05-06]     ┆ [2017-11-25, 2017-11-26,  │\n",
       "│ 2017-11-19, … 2018-… ┆                            ┆                  ┆ … 2018-…                  │\n",
       "└──────────────────────┴────────────────────────────┴──────────────────┴───────────────────────────┘"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.select(\n",
    "    'trending_date',\n",
    "    pl.col('trending_date').list.slice(0, 2).alias('first_2_dates'),\n",
    "    pl.col('trending_date').list.slice(-3, 1).alias('3rd_date_to_last'),\n",
    "    pl.col('trending_date').list.slice(7).alias('from_8th_date_to_end')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>first_and_last</th><th>first_repeated</th><th>first_and_10th_or_null</th></tr><tr><td>list[date]</td><td>list[date]</td><td>list[date]</td><td>list[date]</td></tr></thead><tbody><tr><td>[2018-01-24, 2018-01-25, … 2018-02-12]</td><td>[2018-01-24, 2018-02-12]</td><td>[2018-01-24, 2018-01-24, … 2018-01-24]</td><td>[2018-01-24, 2018-02-12]</td></tr><tr><td>[2017-12-02, 2017-12-03, … 2017-12-07]</td><td>[2017-12-02, 2017-12-07]</td><td>[2017-12-02, 2017-12-02, … 2017-12-02]</td><td>[2017-12-02, null]</td></tr><tr><td>[2017-12-29, 2017-12-30, … 2018-03-03]</td><td>[2017-12-29, 2018-03-03]</td><td>[2017-12-29, 2017-12-29, … 2017-12-29]</td><td>[2017-12-29, 2018-02-09]</td></tr><tr><td>[2018-04-19, 2018-04-20, … 2018-05-14]</td><td>[2018-04-19, 2018-05-14]</td><td>[2018-04-19, 2018-04-19, … 2018-04-19]</td><td>[2018-04-19, 2018-04-29]</td></tr><tr><td>[2017-11-28, 2017-11-29, … 2018-01-05]</td><td>[2017-11-28, 2018-01-05]</td><td>[2017-11-28, 2017-11-28, … 2017-11-28]</td><td>[2017-11-28, 2017-12-24]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌───────────────────────────┬────────────────┬──────────────────────────┬──────────────────────────┐\n",
       "│ trending_date             ┆ first_and_last ┆ first_repeated           ┆ first_and_10th_or_null   │\n",
       "│ ---                       ┆ ---            ┆ ---                      ┆ ---                      │\n",
       "│ list[date]                ┆ list[date]     ┆ list[date]               ┆ list[date]               │\n",
       "╞═══════════════════════════╪════════════════╪══════════════════════════╪══════════════════════════╡\n",
       "│ [2018-01-24, 2018-01-25,  ┆ [2018-01-24,   ┆ [2018-01-24, 2018-01-24, ┆ [2018-01-24, 2018-02-12] │\n",
       "│ … 2018-…                  ┆ 2018-02-12]    ┆ … 2018-…                 ┆                          │\n",
       "│ [2017-12-02, 2017-12-03,  ┆ [2017-12-02,   ┆ [2017-12-02, 2017-12-02, ┆ [2017-12-02, null]       │\n",
       "│ … 2017-…                  ┆ 2017-12-07]    ┆ … 2017-…                 ┆                          │\n",
       "│ [2017-12-29, 2017-12-30,  ┆ [2017-12-29,   ┆ [2017-12-29, 2017-12-29, ┆ [2017-12-29, 2018-02-09] │\n",
       "│ … 2018-…                  ┆ 2018-03-03]    ┆ … 2017-…                 ┆                          │\n",
       "│ [2018-04-19, 2018-04-20,  ┆ [2018-04-19,   ┆ [2018-04-19, 2018-04-19, ┆ [2018-04-19, 2018-04-29] │\n",
       "│ … 2018-…                  ┆ 2018-05-14]    ┆ … 2018-…                 ┆                          │\n",
       "│ [2017-11-28, 2017-11-29,  ┆ [2017-11-28,   ┆ [2017-11-28, 2017-11-28, ┆ [2017-11-28, 2017-12-24] │\n",
       "│ … 2018-…                  ┆ 2018-01-05]    ┆ … 2017-…                 ┆                          │\n",
       "└───────────────────────────┴────────────────┴──────────────────────────┴──────────────────────────┘"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trending_dates_by_channel.select(\n",
    "    'trending_date',\n",
    "    pl.col('trending_date').list.gather([0, -1]).alias('first_and_last'),\n",
    "    pl.col('trending_date').list.gather([0, 0, 0, 0]).alias('first_repeated'),\n",
    "    pl.col('trending_date').list.gather([0, 10], null_on_oob=True).alias('first_and_10th_or_null'),\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>category_id</th><th>category_id_cnt</th><th>category_id_unique</th><th>category_id_unique_cnt</th></tr><tr><td>date</td><td>list[i64]</td><td>u32</td><td>list[i64]</td><td>u32</td></tr></thead><tbody><tr><td>2018-02-13</td><td>[1, 1, … 28]</td><td>199</td><td>[1, 2, … 28]</td><td>14</td></tr><tr><td>2018-03-09</td><td>[1, 1, … 28]</td><td>200</td><td>[1, 10, … 28]</td><td>13</td></tr><tr><td>2018-02-28</td><td>[1, 1, … 28]</td><td>199</td><td>[1, 2, … 28]</td><td>14</td></tr><tr><td>2018-02-19</td><td>[1, 1, … 29]</td><td>200</td><td>[1, 2, … 29]</td><td>15</td></tr><tr><td>2018-03-06</td><td>[1, 1, … 28]</td><td>200</td><td>[1, 2, … 28]</td><td>14</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌───────────────┬──────────────┬─────────────────┬────────────────────┬────────────────────────┐\n",
       "│ trending_date ┆ category_id  ┆ category_id_cnt ┆ category_id_unique ┆ category_id_unique_cnt │\n",
       "│ ---           ┆ ---          ┆ ---             ┆ ---                ┆ ---                    │\n",
       "│ date          ┆ list[i64]    ┆ u32             ┆ list[i64]          ┆ u32                    │\n",
       "╞═══════════════╪══════════════╪═════════════════╪════════════════════╪════════════════════════╡\n",
       "│ 2018-02-13    ┆ [1, 1, … 28] ┆ 199             ┆ [1, 2, … 28]       ┆ 14                     │\n",
       "│ 2018-03-09    ┆ [1, 1, … 28] ┆ 200             ┆ [1, 10, … 28]      ┆ 13                     │\n",
       "│ 2018-02-28    ┆ [1, 1, … 28] ┆ 199             ┆ [1, 2, … 28]       ┆ 14                     │\n",
       "│ 2018-02-19    ┆ [1, 1, … 29] ┆ 200             ┆ [1, 2, … 29]       ┆ 15                     │\n",
       "│ 2018-03-06    ┆ [1, 1, … 28] ┆ 200             ┆ [1, 2, … 28]       ┆ 14                     │\n",
       "└───────────────┴──────────────┴─────────────────┴────────────────────┴────────────────────────┘"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .group_by('trending_date')\n",
    "    .agg('category_id')\n",
    "    .with_columns(pl.col('category_id').list.sort())\n",
    "    .with_columns(\n",
    "        pl.col('category_id'),\n",
    "        pl.col('category_id').list.len().alias('category_id_cnt'),\n",
    "        pl.col('category_id').list.unique().alias('category_id_unique'),\n",
    "        pl.col('category_id').list.unique().list.len().alias('category_id_unique_cnt')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'trending_dates_by_channel' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtrending_dates_by_channel\u001b[49m\u001b[38;5;241m.\u001b[39mwith_columns(\n\u001b[1;32m      2\u001b[0m     pl\u001b[38;5;241m.\u001b[39mcol(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrending_date\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;241m.\u001b[39mlist\u001b[38;5;241m.\u001b[39msample(n\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m, with_replacement\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, seed\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;241m.\u001b[39malias(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msamples\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      5\u001b[0m )\u001b[38;5;241m.\u001b[39mhead()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'trending_dates_by_channel' is not defined"
     ]
    }
   ],
   "source": [
    "trending_dates_by_channel.with_columns(\n",
    "    pl.col(\"trending_date\")\n",
    "    .list.sample(n=3, with_replacement=True, seed=0)\n",
    "    .alias(\"samples\")\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Applying logic to each element in lists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_df = (\n",
    "    df\n",
    "    .group_by('trending_date')\n",
    "    .agg('views', 'channel_title')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>channel_title</th></tr><tr><td>date</td><td>list[i64]</td><td>list[str]</td></tr></thead><tbody><tr><td>2018-02-13</td><td>[266874, 953801, … 736748]</td><td>[&quot;NBC Sports&quot;, &quot;The King of Random&quot;, … &quot;SmarterEveryDay&quot;]</td></tr><tr><td>2018-03-03</td><td>[716096, 1108125, … 1327815]</td><td>[&quot;carrieunderwoodVEVO&quot;, &quot;nigahiga&quot;, … &quot;Brian Hull&quot;]</td></tr><tr><td>2018-02-25</td><td>[237081, 1747075, … 295145]</td><td>[&quot;Thomas Sanders&quot;, &quot;Breakfast Club Power 105.1 FM&quot;, … &quot;Crusoe the Celebrity Dachshund&quot;]</td></tr><tr><td>2018-03-12</td><td>[1611772, 1498364, … 422979]</td><td>[&quot;Saturday Night Live&quot;, &quot;Liza Koshy Too&quot;, … &quot;BBC News&quot;]</td></tr><tr><td>2018-03-09</td><td>[904177, 3008740, … 1159497]</td><td>[&quot;Nintendo&quot;, &quot;CamilaCabelloVEVO&quot;, … &quot;Hannah Stocking&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬──────────────────────────────┬───────────────────────────────────┐\n",
       "│ trending_date ┆ views                        ┆ channel_title                     │\n",
       "│ ---           ┆ ---                          ┆ ---                               │\n",
       "│ date          ┆ list[i64]                    ┆ list[str]                         │\n",
       "╞═══════════════╪══════════════════════════════╪═══════════════════════════════════╡\n",
       "│ 2018-02-13    ┆ [266874, 953801, … 736748]   ┆ [\"NBC Sports\", \"The King of Rand… │\n",
       "│ 2018-03-03    ┆ [716096, 1108125, … 1327815] ┆ [\"carrieunderwoodVEVO\", \"nigahig… │\n",
       "│ 2018-02-25    ┆ [237081, 1747075, … 295145]  ┆ [\"Thomas Sanders\", \"Breakfast Cl… │\n",
       "│ 2018-03-12    ┆ [1611772, 1498364, … 422979] ┆ [\"Saturday Night Live\", \"Liza Ko… │\n",
       "│ 2018-03-09    ┆ [904177, 3008740, … 1159497] ┆ [\"Nintendo\", \"CamilaCabelloVEVO\"… │\n",
       "└───────────────┴──────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>views</th><th>pl.element</th><th>pl.first</th><th>pl.last</th><th>pl.col</th></tr><tr><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[266874, 953801, … 736748]</td><td>[266874, 953801, … 736748]</td><td>[266874, 953801, … 736748]</td><td>[266874, 953801, … 736748]</td><td>[266874, 953801, … 736748]</td></tr><tr><td>[716096, 1108125, … 1327815]</td><td>[716096, 1108125, … 1327815]</td><td>[716096, 1108125, … 1327815]</td><td>[716096, 1108125, … 1327815]</td><td>[716096, 1108125, … 1327815]</td></tr><tr><td>[237081, 1747075, … 295145]</td><td>[237081, 1747075, … 295145]</td><td>[237081, 1747075, … 295145]</td><td>[237081, 1747075, … 295145]</td><td>[237081, 1747075, … 295145]</td></tr><tr><td>[1611772, 1498364, … 422979]</td><td>[1611772, 1498364, … 422979]</td><td>[1611772, 1498364, … 422979]</td><td>[1611772, 1498364, … 422979]</td><td>[1611772, 1498364, … 422979]</td></tr><tr><td>[904177, 3008740, … 1159497]</td><td>[904177, 3008740, … 1159497]</td><td>[904177, 3008740, … 1159497]</td><td>[904177, 3008740, … 1159497]</td><td>[904177, 3008740, … 1159497]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌───────────────────┬───────────────────┬───────────────────┬───────────────────┬──────────────────┐\n",
       "│ views             ┆ pl.element        ┆ pl.first          ┆ pl.last           ┆ pl.col           │\n",
       "│ ---               ┆ ---               ┆ ---               ┆ ---               ┆ ---              │\n",
       "│ list[i64]         ┆ list[i64]         ┆ list[i64]         ┆ list[i64]         ┆ list[i64]        │\n",
       "╞═══════════════════╪═══════════════════╪═══════════════════╪═══════════════════╪══════════════════╡\n",
       "│ [266874, 953801,  ┆ [266874, 953801,  ┆ [266874, 953801,  ┆ [266874, 953801,  ┆ [266874, 953801, │\n",
       "│ … 736748]         ┆ … 736748]         ┆ … 736748]         ┆ … 736748]         ┆ … 736748]        │\n",
       "│ [716096, 1108125, ┆ [716096, 1108125, ┆ [716096, 1108125, ┆ [716096, 1108125, ┆ [716096,         │\n",
       "│ … 1327815]        ┆ … 1327815]        ┆ … 1327815]        ┆ … 1327815]        ┆ 1108125, …       │\n",
       "│                   ┆                   ┆                   ┆                   ┆ 1327815]         │\n",
       "│ [237081, 1747075, ┆ [237081, 1747075, ┆ [237081, 1747075, ┆ [237081, 1747075, ┆ [237081,         │\n",
       "│ … 295145]         ┆ … 295145]         ┆ … 295145]         ┆ … 295145]         ┆ 1747075, …       │\n",
       "│                   ┆                   ┆                   ┆                   ┆ 295145]          │\n",
       "│ [1611772,         ┆ [1611772,         ┆ [1611772,         ┆ [1611772,         ┆ [1611772,        │\n",
       "│ 1498364, …        ┆ 1498364, …        ┆ 1498364, …        ┆ 1498364, …        ┆ 1498364, …       │\n",
       "│ 422979]           ┆ 422979]           ┆ 422979]           ┆ 422979]           ┆ 422979]          │\n",
       "│ [904177, 3008740, ┆ [904177, 3008740, ┆ [904177, 3008740, ┆ [904177, 3008740, ┆ [904177,         │\n",
       "│ … 1159497]        ┆ … 1159497]        ┆ … 1159497]        ┆ … 1159497]        ┆ 3008740, …       │\n",
       "│                   ┆                   ┆                   ┆                   ┆ 1159497]         │\n",
       "└───────────────────┴───────────────────┴───────────────────┴───────────────────┴──────────────────┘"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    agg_df\n",
    "    .select(\n",
    "        'views',\n",
    "        pl.col('views').list.eval(pl.element()).alias('pl.element'),\n",
    "        pl.col('views').list.eval(pl.first()).alias('pl.first'),\n",
    "        pl.col('views').list.eval(pl.last()).alias('pl.last'),\n",
    "        pl.col('views').list.eval(pl.col('')).alias('pl.col')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>channel_title_upper</th></tr><tr><td>list[str]</td><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;NBC Sports&quot;, &quot;The King of Random&quot;]</td><td>[&quot;NBC SPORTS&quot;, &quot;THE KING OF RANDOM&quot;]</td></tr><tr><td>[&quot;carrieunderwoodVEVO&quot;, &quot;nigahiga&quot;]</td><td>[&quot;CARRIEUNDERWOODVEVO&quot;, &quot;NIGAHIGA&quot;]</td></tr><tr><td>[&quot;Thomas Sanders&quot;, &quot;Breakfast Club Power 105.1 FM&quot;]</td><td>[&quot;THOMAS SANDERS&quot;, &quot;BREAKFAST CLUB POWER 105.1 FM&quot;]</td></tr><tr><td>[&quot;Saturday Night Live&quot;, &quot;Liza Koshy Too&quot;]</td><td>[&quot;SATURDAY NIGHT LIVE&quot;, &quot;LIZA KOSHY TOO&quot;]</td></tr><tr><td>[&quot;Nintendo&quot;, &quot;CamilaCabelloVEVO&quot;]</td><td>[&quot;NINTENDO&quot;, &quot;CAMILACABELLOVEVO&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ channel_title                     ┆ channel_title_upper               │\n",
       "│ ---                               ┆ ---                               │\n",
       "│ list[str]                         ┆ list[str]                         │\n",
       "╞═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ [\"NBC Sports\", \"The King of Rand… ┆ [\"NBC SPORTS\", \"THE KING OF RAND… │\n",
       "│ [\"carrieunderwoodVEVO\", \"nigahig… ┆ [\"CARRIEUNDERWOODVEVO\", \"NIGAHIG… │\n",
       "│ [\"Thomas Sanders\", \"Breakfast Cl… ┆ [\"THOMAS SANDERS\", \"BREAKFAST CL… │\n",
       "│ [\"Saturday Night Live\", \"Liza Ko… ┆ [\"SATURDAY NIGHT LIVE\", \"LIZA KO… │\n",
       "│ [\"Nintendo\", \"CamilaCabelloVEVO\"… ┆ [\"NINTENDO\", \"CAMILACABELLOVEVO\"… │\n",
       "└───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "channel_titles_df = (\n",
    "    agg_df\n",
    "    .select(\n",
    "        pl.col(\"channel_title\").list.head(2),\n",
    "        pl.col(\"channel_title\")\n",
    "        .list.eval(pl.element().str.to_uppercase())\n",
    "        .list.head(2)\n",
    "        .alias(\"channel_title_upper\")\n",
    "    )\n",
    ")\n",
    "channel_titles_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channel_title</th><th>channel_title_upper</th></tr><tr><td>list[str]</td><td>list[str]</td></tr></thead><tbody><tr><td>[&quot;NBC Sports&quot;, &quot;The King of Random&quot;]</td><td>[&quot;THE KING OF RANDOM&quot;]</td></tr><tr><td>[&quot;carrieunderwoodVEVO&quot;, &quot;nigahiga&quot;]</td><td>[&quot;CARRIEUNDERWOODVEVO&quot;, &quot;NIGAHIGA&quot;]</td></tr><tr><td>[&quot;Thomas Sanders&quot;, &quot;Breakfast Club Power 105.1 FM&quot;]</td><td>[&quot;THOMAS SANDERS&quot;, &quot;BREAKFAST CLUB POWER 105.1 FM&quot;]</td></tr><tr><td>[&quot;Saturday Night Live&quot;, &quot;Liza Koshy Too&quot;]</td><td>[&quot;SATURDAY NIGHT LIVE&quot;, &quot;LIZA KOSHY TOO&quot;]</td></tr><tr><td>[&quot;Nintendo&quot;, &quot;CamilaCabelloVEVO&quot;]</td><td>[&quot;CAMILACABELLOVEVO&quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ channel_title                     ┆ channel_title_upper               │\n",
       "│ ---                               ┆ ---                               │\n",
       "│ list[str]                         ┆ list[str]                         │\n",
       "╞═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ [\"NBC Sports\", \"The King of Rand… ┆ [\"THE KING OF RANDOM\"]            │\n",
       "│ [\"carrieunderwoodVEVO\", \"nigahig… ┆ [\"CARRIEUNDERWOODVEVO\", \"NIGAHIG… │\n",
       "│ [\"Thomas Sanders\", \"Breakfast Cl… ┆ [\"THOMAS SANDERS\", \"BREAKFAST CL… │\n",
       "│ [\"Saturday Night Live\", \"Liza Ko… ┆ [\"SATURDAY NIGHT LIVE\", \"LIZA KO… │\n",
       "│ [\"Nintendo\", \"CamilaCabelloVEVO\"… ┆ [\"CAMILACABELLOVEVO\"]             │\n",
       "└───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    channel_titles_df\n",
    "    .with_columns(\n",
    "        pl.col('channel_title_upper')\n",
    "        .list.eval(\n",
    "            pl.element().filter(\n",
    "                pl.element().str.contains('A', literal=True)\n",
    "            )\n",
    "        )\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>views_rank</th></tr><tr><td>date</td><td>list[i64]</td><td>list[u32]</td></tr></thead><tbody><tr><td>2018-02-13</td><td>[266874, 953801, … 736748]</td><td>[133, 66, … 79]</td></tr><tr><td>2018-03-03</td><td>[716096, 1108125, … 1327815]</td><td>[80, 55, … 47]</td></tr><tr><td>2018-02-25</td><td>[237081, 1747075, … 295145]</td><td>[115, 35, … 109]</td></tr><tr><td>2018-03-12</td><td>[1611772, 1498364, … 422979]</td><td>[54, 61, … 132]</td></tr><tr><td>2018-03-09</td><td>[904177, 3008740, … 1159497]</td><td>[75, 18, … 65]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬──────────────────────────────┬──────────────────┐\n",
       "│ trending_date ┆ views                        ┆ views_rank       │\n",
       "│ ---           ┆ ---                          ┆ ---              │\n",
       "│ date          ┆ list[i64]                    ┆ list[u32]        │\n",
       "╞═══════════════╪══════════════════════════════╪══════════════════╡\n",
       "│ 2018-02-13    ┆ [266874, 953801, … 736748]   ┆ [133, 66, … 79]  │\n",
       "│ 2018-03-03    ┆ [716096, 1108125, … 1327815] ┆ [80, 55, … 47]   │\n",
       "│ 2018-02-25    ┆ [237081, 1747075, … 295145]  ┆ [115, 35, … 109] │\n",
       "│ 2018-03-12    ┆ [1611772, 1498364, … 422979] ┆ [54, 61, … 132]  │\n",
       "│ 2018-03-09    ┆ [904177, 3008740, … 1159497] ┆ [75, 18, … 65]   │\n",
       "└───────────────┴──────────────────────────────┴──────────────────┘"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    agg_df\n",
    "    .select(\n",
    "        'trending_date',\n",
    "        'views',\n",
    "        pl.col('views')\n",
    "        .list.eval(pl.element().rank('dense', descending=True))\n",
    "        .alias('views_rank'),\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>views_rank</th></tr><tr><td>date</td><td>list[i64]</td><td>list[u32]</td></tr></thead><tbody><tr><td>2018-02-13</td><td>[266874, 953801, … 736748]</td><td>[133, 66, … 79]</td></tr><tr><td>2018-03-03</td><td>[716096, 1108125, … 1327815]</td><td>[80, 55, … 47]</td></tr><tr><td>2018-02-25</td><td>[237081, 1747075, … 295145]</td><td>[115, 35, … 109]</td></tr><tr><td>2018-03-12</td><td>[1611772, 1498364, … 422979]</td><td>[54, 61, … 132]</td></tr><tr><td>2018-03-09</td><td>[904177, 3008740, … 1159497]</td><td>[75, 18, … 65]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬──────────────────────────────┬──────────────────┐\n",
       "│ trending_date ┆ views                        ┆ views_rank       │\n",
       "│ ---           ┆ ---                          ┆ ---              │\n",
       "│ date          ┆ list[i64]                    ┆ list[u32]        │\n",
       "╞═══════════════╪══════════════════════════════╪══════════════════╡\n",
       "│ 2018-02-13    ┆ [266874, 953801, … 736748]   ┆ [133, 66, … 79]  │\n",
       "│ 2018-03-03    ┆ [716096, 1108125, … 1327815] ┆ [80, 55, … 47]   │\n",
       "│ 2018-02-25    ┆ [237081, 1747075, … 295145]  ┆ [115, 35, … 109] │\n",
       "│ 2018-03-12    ┆ [1611772, 1498364, … 422979] ┆ [54, 61, … 132]  │\n",
       "│ 2018-03-09    ┆ [904177, 3008740, … 1159497] ┆ [75, 18, … 65]   │\n",
       "└───────────────┴──────────────────────────────┴──────────────────┘"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "views_rank_df = (\n",
    "    agg_df\n",
    "    .select(\n",
    "        'trending_date',\n",
    "        'views',\n",
    "        pl.col('views')\n",
    "        .list.eval(pl.element().rank('dense', descending=True))\n",
    "        .alias('views_rank'),\n",
    "    )\n",
    ")\n",
    "views_rank_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>views_rank</th></tr><tr><td>date</td><td>list[i64]</td><td>list[u32]</td></tr></thead><tbody><tr><td>2017-11-29</td><td>[30583293, 6969128, 19670964]</td><td>[1, 3, 2]</td></tr><tr><td>2018-06-02</td><td>[32523416, 42700273, 225211923]</td><td>[3, 2, 1]</td></tr><tr><td>2018-04-19</td><td>[52556278, 36908726, 39814522]</td><td>[1, 3, 2]</td></tr><tr><td>2018-05-21</td><td>[65396157, 162556776, 60207478]</td><td>[2, 1, 3]</td></tr><tr><td>2018-02-03</td><td>[15643726, 19392316, 10878906]</td><td>[2, 1, 3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬─────────────────────────────────┬────────────┐\n",
       "│ trending_date ┆ views                           ┆ views_rank │\n",
       "│ ---           ┆ ---                             ┆ ---        │\n",
       "│ date          ┆ list[i64]                       ┆ list[u32]  │\n",
       "╞═══════════════╪═════════════════════════════════╪════════════╡\n",
       "│ 2017-11-29    ┆ [30583293, 6969128, 19670964]   ┆ [1, 3, 2]  │\n",
       "│ 2018-06-02    ┆ [32523416, 42700273, 225211923] ┆ [3, 2, 1]  │\n",
       "│ 2018-04-19    ┆ [52556278, 36908726, 39814522]  ┆ [1, 3, 2]  │\n",
       "│ 2018-05-21    ┆ [65396157, 162556776, 60207478] ┆ [2, 1, 3]  │\n",
       "│ 2018-02-03    ┆ [15643726, 19392316, 10878906]  ┆ [2, 1, 3]  │\n",
       "└───────────────┴─────────────────────────────────┴────────────┘"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top3_views_df = (\n",
    "    views_rank_df\n",
    "    .explode('views', 'views_rank')\n",
    "    .filter(pl.col('views_rank')<=3)\n",
    "    .group_by('trending_date')\n",
    "    .agg(pl.all())\n",
    ")\n",
    "top3_views_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>views_rank</th></tr><tr><td>date</td><td>i64</td><td>u32</td></tr></thead><tbody><tr><td>2018-02-13</td><td>266874</td><td>133</td></tr><tr><td>2018-02-13</td><td>953801</td><td>66</td></tr><tr><td>2018-02-13</td><td>638618</td><td>85</td></tr><tr><td>2018-02-13</td><td>319444</td><td>122</td></tr><tr><td>2018-02-13</td><td>3780988</td><td>23</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬─────────┬────────────┐\n",
       "│ trending_date ┆ views   ┆ views_rank │\n",
       "│ ---           ┆ ---     ┆ ---        │\n",
       "│ date          ┆ i64     ┆ u32        │\n",
       "╞═══════════════╪═════════╪════════════╡\n",
       "│ 2018-02-13    ┆ 266874  ┆ 133        │\n",
       "│ 2018-02-13    ┆ 953801  ┆ 66         │\n",
       "│ 2018-02-13    ┆ 638618  ┆ 85         │\n",
       "│ 2018-02-13    ┆ 319444  ┆ 122        │\n",
       "│ 2018-02-13    ┆ 3780988 ┆ 23         │\n",
       "└───────────────┴─────────┴────────────┘"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    views_rank_df\n",
    "    .explode('views', 'views_rank')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>views_rank</th><th>views</th><th>diff from the most views</th></tr><tr><td>list[u32]</td><td>list[i64]</td><td>list[i64]</td></tr></thead><tbody><tr><td>[1, 3, 2]</td><td>[30583293, 6969128, 19670964]</td><td>[0, 23614165, 10912329]</td></tr><tr><td>[3, 2, 1]</td><td>[32523416, 42700273, 225211923]</td><td>[192688507, 182511650, 0]</td></tr><tr><td>[1, 3, 2]</td><td>[52556278, 36908726, 39814522]</td><td>[0, 15647552, 12741756]</td></tr><tr><td>[2, 1, 3]</td><td>[65396157, 162556776, 60207478]</td><td>[97160619, 0, 102349298]</td></tr><tr><td>[2, 1, 3]</td><td>[15643726, 19392316, 10878906]</td><td>[3748590, 0, 8513410]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌────────────┬─────────────────────────────────┬───────────────────────────┐\n",
       "│ views_rank ┆ views                           ┆ diff from the most views  │\n",
       "│ ---        ┆ ---                             ┆ ---                       │\n",
       "│ list[u32]  ┆ list[i64]                       ┆ list[i64]                 │\n",
       "╞════════════╪═════════════════════════════════╪═══════════════════════════╡\n",
       "│ [1, 3, 2]  ┆ [30583293, 6969128, 19670964]   ┆ [0, 23614165, 10912329]   │\n",
       "│ [3, 2, 1]  ┆ [32523416, 42700273, 225211923] ┆ [192688507, 182511650, 0] │\n",
       "│ [1, 3, 2]  ┆ [52556278, 36908726, 39814522]  ┆ [0, 15647552, 12741756]   │\n",
       "│ [2, 1, 3]  ┆ [65396157, 162556776, 60207478] ┆ [97160619, 0, 102349298]  │\n",
       "│ [2, 1, 3]  ┆ [15643726, 19392316, 10878906]  ┆ [3748590, 0, 8513410]     │\n",
       "└────────────┴─────────────────────────────────┴───────────────────────────┘"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    top3_views_df\n",
    "    .select(\n",
    "        'views_rank',\n",
    "        'views',\n",
    "        pl.col('views')\n",
    "        .list.eval(pl.element().max() - pl.element())\n",
    "        .alias('diff from the most views')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>trending_date</th><th>views</th><th>views_rank</th></tr><tr><td>date</td><td>list[i64]</td><td>list[u32]</td></tr></thead><tbody><tr><td>2017-11-29</td><td>[30583293, 6969128, 19670964]</td><td>[1, 3, 2]</td></tr><tr><td>2018-06-02</td><td>[32523416, 42700273, 225211923]</td><td>[3, 2, 1]</td></tr><tr><td>2018-04-19</td><td>[52556278, 36908726, 39814522]</td><td>[1, 3, 2]</td></tr><tr><td>2018-05-21</td><td>[65396157, 162556776, 60207478]</td><td>[2, 1, 3]</td></tr><tr><td>2018-02-03</td><td>[15643726, 19392316, 10878906]</td><td>[2, 1, 3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────┬─────────────────────────────────┬────────────┐\n",
       "│ trending_date ┆ views                           ┆ views_rank │\n",
       "│ ---           ┆ ---                             ┆ ---        │\n",
       "│ date          ┆ list[i64]                       ┆ list[u32]  │\n",
       "╞═══════════════╪═════════════════════════════════╪════════════╡\n",
       "│ 2017-11-29    ┆ [30583293, 6969128, 19670964]   ┆ [1, 3, 2]  │\n",
       "│ 2018-06-02    ┆ [32523416, 42700273, 225211923] ┆ [3, 2, 1]  │\n",
       "│ 2018-04-19    ┆ [52556278, 36908726, 39814522]  ┆ [1, 3, 2]  │\n",
       "│ 2018-05-21    ┆ [65396157, 162556776, 60207478] ┆ [2, 1, 3]  │\n",
       "│ 2018-02-03    ┆ [15643726, 19392316, 10878906]  ┆ [2, 1, 3]  │\n",
       "└───────────────┴─────────────────────────────────┴────────────┘"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top3_views_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>views_rank_1</th><th>views_rank_2</th><th>intersection</th><th>union</th><th>difference</th><th>symmetric_difference</th></tr><tr><td>list[u32]</td><td>list[u32]</td><td>list[u32]</td><td>list[u32]</td><td>list[u32]</td><td>list[u32]</td></tr></thead><tbody><tr><td>[1, 3]</td><td>[3, 2]</td><td>[3]</td><td>[1, 3, 2]</td><td>[1]</td><td>[1, 2]</td></tr><tr><td>[3, 2]</td><td>[2, 1]</td><td>[2]</td><td>[3, 2, 1]</td><td>[3]</td><td>[3, 1]</td></tr><tr><td>[1, 3]</td><td>[3, 2]</td><td>[3]</td><td>[1, 3, 2]</td><td>[1]</td><td>[1, 2]</td></tr><tr><td>[2, 1]</td><td>[1, 3]</td><td>[1]</td><td>[2, 1, 3]</td><td>[2]</td><td>[2, 3]</td></tr><tr><td>[2, 1]</td><td>[1, 3]</td><td>[1]</td><td>[2, 1, 3]</td><td>[2]</td><td>[2, 3]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌──────────────┬──────────────┬──────────────┬───────────┬────────────┬──────────────────────┐\n",
       "│ views_rank_1 ┆ views_rank_2 ┆ intersection ┆ union     ┆ difference ┆ symmetric_difference │\n",
       "│ ---          ┆ ---          ┆ ---          ┆ ---       ┆ ---        ┆ ---                  │\n",
       "│ list[u32]    ┆ list[u32]    ┆ list[u32]    ┆ list[u32] ┆ list[u32]  ┆ list[u32]            │\n",
       "╞══════════════╪══════════════╪══════════════╪═══════════╪════════════╪══════════════════════╡\n",
       "│ [1, 3]       ┆ [3, 2]       ┆ [3]          ┆ [1, 3, 2] ┆ [1]        ┆ [1, 2]               │\n",
       "│ [3, 2]       ┆ [2, 1]       ┆ [2]          ┆ [3, 2, 1] ┆ [3]        ┆ [3, 1]               │\n",
       "│ [1, 3]       ┆ [3, 2]       ┆ [3]          ┆ [1, 3, 2] ┆ [1]        ┆ [1, 2]               │\n",
       "│ [2, 1]       ┆ [1, 3]       ┆ [1]          ┆ [2, 1, 3] ┆ [2]        ┆ [2, 3]               │\n",
       "│ [2, 1]       ┆ [1, 3]       ┆ [1]          ┆ [2, 1, 3] ┆ [2]        ┆ [2, 3]               │\n",
       "└──────────────┴──────────────┴──────────────┴───────────┴────────────┴──────────────────────┘"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    top3_views_df\n",
    "    .with_columns(\n",
    "        pl.col('views_rank').list.slice(0, 2).alias('views_rank_1'),\n",
    "        pl.col('views_rank').list.slice(-2, 2).alias('views_rank_2')\n",
    "    )\n",
    "    .select(\n",
    "        'views_rank_1',\n",
    "        'views_rank_2',\n",
    "        pl.col('views_rank_1').list.set_intersection('views_rank_2')\n",
    "        .alias('intersection'),\n",
    "        pl.col('views_rank_1').list.set_union('views_rank_2')\n",
    "        .alias('union'),\n",
    "        pl.col('views_rank_1').list.set_difference('views_rank_2')\n",
    "        .alias('difference'),\n",
    "        pl.col('views_rank_1').list.set_symmetric_difference('views_rank_2')\n",
    "        .alias('symmetric_difference'),\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with structs and JSON data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 16)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>visitorId</th><th>visitNumber</th><th>visitId</th><th>visitStartTime</th><th>date</th><th>totals</th><th>trafficSource</th><th>device</th><th>geoNetwork</th><th>customDimensions</th><th>hits</th><th>fullVisitorId</th><th>userId</th><th>clientId</th><th>channelGrouping</th><th>socialEngagementType</th></tr><tr><td>null</td><td>str</td><td>str</td><td>str</td><td>str</td><td>struct[13]</td><td>struct[9]</td><td>struct[17]</td><td>struct[11]</td><td>list[struct[2]]</td><td>list[struct[33]]</td><td>str</td><td>null</td><td>null</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>null</td><td>&quot;1&quot;</td><td>&quot;1501591568&quot;</td><td>&quot;1501591568&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{null,&quot;(not set)&quot;,&quot;(direct)&quot;,&quot;(none)&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>{&quot;Chrome&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;Windows&quot;,&quot;not available in demo dataset&quot;,&quot;false&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,null,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;desktop&quot;}</td><td>{&quot;Europe&quot;,&quot;Southern Europe&quot;,&quot;Greece&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;tellas.gr&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;}</td><td>[]</td><td>[{{&quot;0&quot;,&quot;1&quot;,null},null,{&quot;(not set)&quot;,&quot;Bags&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,null,&quot;1&quot;,null,null,null},&quot;1&quot;,null,{null,null,null,null,null,null,null,null,&quot;shop.googlemerchandisestore.com/google+redesign/bags/google+zipper+front+sports+bag.axd&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/bags/google+zipper+front+sports+bag.axd&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/bags/google+zipper+front+sports+bag.axd&quot;,&quot;0&quot;},[],&quot;PAGE&quot;,[],&quot;true&quot;,&quot;https://www.google.gr/&quot;,null,&quot;web&quot;,&quot;46&quot;,&quot;true&quot;,null,null,{null,&quot;true&quot;,null,null},null,{null,null,null,null,&quot;(not set)&quot;,null,&quot;No&quot;,&quot; : &quot;},[],[],[],null,&quot;5&quot;,null,{&quot;/google+redesign/bags/google+zipper+front+sports+bag.axd&quot;,&quot;shop.googlemerchandisestore.com&quot;,&quot;Page Unavailable&quot;,null,null,&quot;/google+redesign/&quot;,&quot;/bags/&quot;,&quot;/google+zipper+front+sports+bag.axd&quot;,&quot;&quot;},[],[],&quot;true&quot;,null,null,&quot;0&quot;}]</td><td>&quot;34183340117798…</td><td>null</td><td>null</td><td>&quot;Organic Search…</td><td>&quot;Not Socially E…</td></tr><tr><td>null</td><td>&quot;2&quot;</td><td>&quot;1501589647&quot;</td><td>&quot;1501589647&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,null,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>{&quot;Chrome&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;Windows&quot;,&quot;not available in demo dataset&quot;,&quot;false&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,null,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;desktop&quot;}</td><td>{&quot;Asia&quot;,&quot;Southern Asia&quot;,&quot;India&quot;,&quot;Maharashtra&quot;,&quot;(not set)&quot;,&quot;Mumbai&quot;,&quot;not available in demo dataset&quot;,&quot;unknown.unknown&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;}</td><td>[{&quot;4&quot;,&quot;APAC&quot;}]</td><td>[{{&quot;0&quot;,&quot;1&quot;,null},null,{&quot;(not set)&quot;,&quot;Brands&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,null,&quot;1&quot;,null,null,null},&quot;1&quot;,null,{null,null,null,null,null,null,null,null,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;0&quot;},[],&quot;PAGE&quot;,[],&quot;true&quot;,&quot;https://analytics.google.com/analytics/web/&quot;,null,&quot;web&quot;,&quot;14&quot;,&quot;true&quot;,null,null,{null,&quot;true&quot;,null,null},null,{null,null,null,null,&quot;(not set)&quot;,null,&quot;No&quot;,&quot; : &quot;},[],[],[],null,&quot;5&quot;,null,{&quot;/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com&quot;,&quot;Page Unavailable&quot;,null,null,&quot;/google+redesign/&quot;,&quot;/shop+by+brand/&quot;,&quot;/youtube&quot;,&quot;&quot;},[],[],&quot;true&quot;,null,null,&quot;0&quot;}]</td><td>&quot;24743978550413…</td><td>null</td><td>null</td><td>&quot;Referral&quot;</td><td>&quot;Not Socially E…</td></tr><tr><td>null</td><td>&quot;1&quot;</td><td>&quot;1501616621&quot;</td><td>&quot;1501616621&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>{&quot;Chrome&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;Windows&quot;,&quot;not available in demo dataset&quot;,&quot;false&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,null,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;desktop&quot;}</td><td>{&quot;Europe&quot;,&quot;Northern Europe&quot;,&quot;United Kingdom&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;as9105.com&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;}</td><td>[{&quot;4&quot;,&quot;EMEA&quot;}]</td><td>[{{&quot;0&quot;,&quot;1&quot;,null},null,{&quot;(not set)&quot;,&quot;Brands&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,null,&quot;1&quot;,null,null,null},&quot;1&quot;,null,{null,null,null,null,null,null,null,null,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;0&quot;},[],&quot;PAGE&quot;,[],&quot;true&quot;,&quot;https://analytics.google.com/analytics/web/?utm_source=demoaccount&amp;utm_medium=demoaccount&amp;utm_campaign=demoaccount&quot;,null,&quot;web&quot;,&quot;43&quot;,&quot;true&quot;,null,null,{null,&quot;true&quot;,null,null},null,{null,null,null,null,&quot;(not set)&quot;,null,&quot;No&quot;,&quot; : &quot;},[],[],[],null,&quot;12&quot;,null,{&quot;/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com&quot;,&quot;Page Unavailable&quot;,null,null,&quot;/google+redesign/&quot;,&quot;/shop+by+brand/&quot;,&quot;/youtube&quot;,&quot;&quot;},[],[],&quot;true&quot;,null,null,&quot;0&quot;}]</td><td>&quot;58704628207131…</td><td>null</td><td>null</td><td>&quot;Referral&quot;</td><td>&quot;Not Socially E…</td></tr><tr><td>null</td><td>&quot;1&quot;</td><td>&quot;1501601200&quot;</td><td>&quot;1501601200&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>{&quot;Firefox&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;Windows&quot;,&quot;not available in demo dataset&quot;,&quot;false&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,null,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;desktop&quot;}</td><td>{&quot;Americas&quot;,&quot;Northern America&quot;,&quot;United States&quot;,&quot;Texas&quot;,&quot;Dallas-Ft. Worth TX&quot;,&quot;Dallas&quot;,&quot;not available in demo dataset&quot;,&quot;h5colo.com&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td><td>[{{&quot;0&quot;,&quot;1&quot;,null},null,{&quot;(not set)&quot;,&quot;Brands&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,null,&quot;1&quot;,null,null,null},&quot;1&quot;,null,{null,null,null,null,null,null,null,null,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;0&quot;},[],&quot;PAGE&quot;,[],&quot;true&quot;,&quot;https://analytics.google.com/analytics/web/&quot;,null,&quot;web&quot;,&quot;26&quot;,&quot;true&quot;,null,null,{null,&quot;true&quot;,null,null},null,{null,null,null,null,&quot;(not set)&quot;,null,&quot;No&quot;,&quot; : &quot;},[],[],[],null,&quot;8&quot;,null,{&quot;/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com&quot;,&quot;Page Unavailable&quot;,null,null,&quot;/google+redesign/&quot;,&quot;/shop+by+brand/&quot;,&quot;/youtube&quot;,&quot;&quot;},[],[],&quot;true&quot;,null,null,&quot;0&quot;}]</td><td>&quot;93978091713494…</td><td>null</td><td>null</td><td>&quot;Referral&quot;</td><td>&quot;Not Socially E…</td></tr><tr><td>null</td><td>&quot;1&quot;</td><td>&quot;1501615525&quot;</td><td>&quot;1501615525&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;adwords.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>{&quot;Chrome&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;Windows&quot;,&quot;not available in demo dataset&quot;,&quot;false&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,null,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;desktop&quot;}</td><td>{&quot;Americas&quot;,&quot;Northern America&quot;,&quot;United States&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;(not set)&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;,&quot;not available in demo dataset&quot;}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td><td>[{{&quot;0&quot;,&quot;1&quot;,null},null,{&quot;(not set)&quot;,&quot;Brands&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(not set)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,&quot;(entrance)&quot;,null,&quot;1&quot;,null,null,null},&quot;1&quot;,null,{null,null,null,null,null,null,null,null,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com/google+redesign/shop+by+brand/youtube&quot;,&quot;0&quot;},[],&quot;PAGE&quot;,[],&quot;true&quot;,&quot;https://adwords.google.com/analytics/web/?hl=en_US&amp;__o=cues&amp;authuser=0&quot;,null,&quot;web&quot;,&quot;25&quot;,&quot;true&quot;,null,null,{null,&quot;true&quot;,null,null},null,{null,null,null,null,&quot;(not set)&quot;,null,&quot;No&quot;,&quot; : &quot;},[],[],[],null,&quot;12&quot;,null,{&quot;/google+redesign/shop+by+brand/youtube&quot;,&quot;shop.googlemerchandisestore.com&quot;,&quot;Page Unavailable&quot;,null,null,&quot;/google+redesign/&quot;,&quot;/shop+by+brand/&quot;,&quot;/youtube&quot;,&quot;&quot;},[],[],&quot;true&quot;,null,null,&quot;0&quot;}]</td><td>&quot;60899029431845…</td><td>null</td><td>null</td><td>&quot;Referral&quot;</td><td>&quot;Not Socially E…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 16)\n",
       "┌───────────┬────────────┬────────────┬────────────┬───┬────────┬──────────┬───────────┬───────────┐\n",
       "│ visitorId ┆ visitNumbe ┆ visitId    ┆ visitStart ┆ … ┆ userId ┆ clientId ┆ channelGr ┆ socialEng │\n",
       "│ ---       ┆ r          ┆ ---        ┆ Time       ┆   ┆ ---    ┆ ---      ┆ ouping    ┆ agementTy │\n",
       "│ null      ┆ ---        ┆ str        ┆ ---        ┆   ┆ null   ┆ null     ┆ ---       ┆ pe        │\n",
       "│           ┆ str        ┆            ┆ str        ┆   ┆        ┆          ┆ str       ┆ ---       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ str       │\n",
       "╞═══════════╪════════════╪════════════╪════════════╪═══╪════════╪══════════╪═══════════╪═══════════╡\n",
       "│ null      ┆ 1          ┆ 1501591568 ┆ 1501591568 ┆ … ┆ null   ┆ null     ┆ Organic   ┆ Not       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆ Search    ┆ Socially  │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Engaged   │\n",
       "│ null      ┆ 2          ┆ 1501589647 ┆ 1501589647 ┆ … ┆ null   ┆ null     ┆ Referral  ┆ Not       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Socially  │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Engaged   │\n",
       "│ null      ┆ 1          ┆ 1501616621 ┆ 1501616621 ┆ … ┆ null   ┆ null     ┆ Referral  ┆ Not       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Socially  │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Engaged   │\n",
       "│ null      ┆ 1          ┆ 1501601200 ┆ 1501601200 ┆ … ┆ null   ┆ null     ┆ Referral  ┆ Not       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Socially  │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Engaged   │\n",
       "│ null      ┆ 1          ┆ 1501615525 ┆ 1501615525 ┆ … ┆ null   ┆ null     ┆ Referral  ┆ Not       │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Socially  │\n",
       "│           ┆            ┆            ┆            ┆   ┆        ┆          ┆           ┆ Engaged   │\n",
       "└───────────┴────────────┴────────────┴────────────┴───┴────────┴──────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_json('../data/ga_20170801.json')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>visitId</th><th>date</th><th>totals</th><th>trafficSource</th><th>customDimensions</th><th>channelGrouping</th></tr><tr><td>str</td><td>str</td><td>struct[13]</td><td>struct[9]</td><td>list[struct[2]]</td><td>str</td></tr></thead><tbody><tr><td>&quot;1501591568&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{null,&quot;(not set)&quot;,&quot;(direct)&quot;,&quot;(none)&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[]</td><td>&quot;Organic Search…</td></tr><tr><td>&quot;1501589647&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,null,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;APAC&quot;}]</td><td>&quot;Referral&quot;</td></tr><tr><td>&quot;1501616621&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;EMEA&quot;}]</td><td>&quot;Referral&quot;</td></tr><tr><td>&quot;1501601200&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td><td>&quot;Referral&quot;</td></tr><tr><td>&quot;1501615525&quot;</td><td>&quot;20170801&quot;</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;adwords.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td><td>&quot;Referral&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌────────────┬──────────┬──────────────────┬──────────────────┬──────────────────┬─────────────────┐\n",
       "│ visitId    ┆ date     ┆ totals           ┆ trafficSource    ┆ customDimensions ┆ channelGrouping │\n",
       "│ ---        ┆ ---      ┆ ---              ┆ ---              ┆ ---              ┆ ---             │\n",
       "│ str        ┆ str      ┆ struct[13]       ┆ struct[9]        ┆ list[struct[2]]  ┆ str             │\n",
       "╞════════════╪══════════╪══════════════════╪══════════════════╪══════════════════╪═════════════════╡\n",
       "│ 1501591568 ┆ 20170801 ┆ {\"1\",\"1\",\"1\",nul ┆ {null,\"(not set) ┆ []               ┆ Organic Search  │\n",
       "│            ┆          ┆ l,\"1\",null,null, ┆ \",\"(direct)\",\"(n ┆                  ┆                 │\n",
       "│            ┆          ┆ …                ┆ …                ┆                  ┆                 │\n",
       "│ 1501589647 ┆ 20170801 ┆ {\"1\",\"1\",\"1\",nul ┆ {\"/analytics/web ┆ [{\"4\",\"APAC\"}]   ┆ Referral        │\n",
       "│            ┆          ┆ l,\"1\",null,null, ┆ /\",\"(not         ┆                  ┆                 │\n",
       "│            ┆          ┆ …                ┆ set)\",\"…         ┆                  ┆                 │\n",
       "│ 1501616621 ┆ 20170801 ┆ {\"1\",\"1\",\"1\",nul ┆ {\"/analytics/web ┆ [{\"4\",\"EMEA\"}]   ┆ Referral        │\n",
       "│            ┆          ┆ l,\"1\",null,null, ┆ /\",\"(not         ┆                  ┆                 │\n",
       "│            ┆          ┆ …                ┆ set)\",\"…         ┆                  ┆                 │\n",
       "│ 1501601200 ┆ 20170801 ┆ {\"1\",\"1\",\"1\",nul ┆ {\"/analytics/web ┆ [{\"4\",\"North     ┆ Referral        │\n",
       "│            ┆          ┆ l,\"1\",null,null, ┆ /\",\"(not         ┆ America\"}]       ┆                 │\n",
       "│            ┆          ┆ …                ┆ set)\",\"…         ┆                  ┆                 │\n",
       "│ 1501615525 ┆ 20170801 ┆ {\"1\",\"1\",\"1\",nul ┆ {\"/analytics/web ┆ [{\"4\",\"North     ┆ Referral        │\n",
       "│            ┆          ┆ l,\"1\",null,null, ┆ /\",\"(not         ┆ America\"}]       ┆                 │\n",
       "│            ┆          ┆ …                ┆ set)\",\"…         ┆                  ┆                 │\n",
       "└────────────┴──────────┴──────────────────┴──────────────────┴──────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = ['visitId', 'date', 'totals', 'trafficSource', 'customDimensions', 'channelGrouping']\n",
    "df = df.select(cols)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>totals</th><th>trafficSource</th><th>customDimensions</th></tr><tr><td>struct[13]</td><td>struct[9]</td><td>list[struct[2]]</td></tr></thead><tbody><tr><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{null,&quot;(not set)&quot;,&quot;(direct)&quot;,&quot;(none)&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[]</td></tr><tr><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,null,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;APAC&quot;}]</td></tr><tr><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;EMEA&quot;}]</td></tr><tr><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;analytics.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td></tr><tr><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td><td>{&quot;/analytics/web/&quot;,&quot;(not set)&quot;,&quot;adwords.google.com&quot;,&quot;referral&quot;,null,null,{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null},null,null}</td><td>[{&quot;4&quot;,&quot;North America&quot;}]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌───────────────────────────────────┬───────────────────────────────────┬─────────────────────────┐\n",
       "│ totals                            ┆ trafficSource                     ┆ customDimensions        │\n",
       "│ ---                               ┆ ---                               ┆ ---                     │\n",
       "│ struct[13]                        ┆ struct[9]                         ┆ list[struct[2]]         │\n",
       "╞═══════════════════════════════════╪═══════════════════════════════════╪═════════════════════════╡\n",
       "│ {\"1\",\"1\",\"1\",null,\"1\",null,null,… ┆ {null,\"(not set)\",\"(direct)\",\"(n… ┆ []                      │\n",
       "│ {\"1\",\"1\",\"1\",null,\"1\",null,null,… ┆ {\"/analytics/web/\",\"(not set)\",\"… ┆ [{\"4\",\"APAC\"}]          │\n",
       "│ {\"1\",\"1\",\"1\",null,\"1\",null,null,… ┆ {\"/analytics/web/\",\"(not set)\",\"… ┆ [{\"4\",\"EMEA\"}]          │\n",
       "│ {\"1\",\"1\",\"1\",null,\"1\",null,null,… ┆ {\"/analytics/web/\",\"(not set)\",\"… ┆ [{\"4\",\"North America\"}] │\n",
       "│ {\"1\",\"1\",\"1\",null,\"1\",null,null,… ┆ {\"/analytics/web/\",\"(not set)\",\"… ┆ [{\"4\",\"North America\"}] │\n",
       "└───────────────────────────────────┴───────────────────────────────────┴─────────────────────────┘"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('totals', 'trafficSource', 'customDimensions').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.with_columns(\n",
    "    pl.struct('visitId', 'date', 'channelGrouping').alias('structFromCols')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>visitId</th><th>date</th><th>channelGrouping</th><th>structFromCols</th></tr><tr><td>str</td><td>str</td><td>str</td><td>struct[3]</td></tr></thead><tbody><tr><td>&quot;1501591568&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Organic Search…</td><td>{&quot;1501591568&quot;,&quot;20170801&quot;,&quot;Organic Search&quot;}</td></tr><tr><td>&quot;1501589647&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td><td>{&quot;1501589647&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td></tr><tr><td>&quot;1501616621&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td><td>{&quot;1501616621&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td></tr><tr><td>&quot;1501601200&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td><td>{&quot;1501601200&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td></tr><tr><td>&quot;1501615525&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td><td>{&quot;1501615525&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌────────────┬──────────┬─────────────────┬───────────────────────────────────┐\n",
       "│ visitId    ┆ date     ┆ channelGrouping ┆ structFromCols                    │\n",
       "│ ---        ┆ ---      ┆ ---             ┆ ---                               │\n",
       "│ str        ┆ str      ┆ str             ┆ struct[3]                         │\n",
       "╞════════════╪══════════╪═════════════════╪═══════════════════════════════════╡\n",
       "│ 1501591568 ┆ 20170801 ┆ Organic Search  ┆ {\"1501591568\",\"20170801\",\"Organi… │\n",
       "│ 1501589647 ┆ 20170801 ┆ Referral        ┆ {\"1501589647\",\"20170801\",\"Referr… │\n",
       "│ 1501616621 ┆ 20170801 ┆ Referral        ┆ {\"1501616621\",\"20170801\",\"Referr… │\n",
       "│ 1501601200 ┆ 20170801 ┆ Referral        ┆ {\"1501601200\",\"20170801\",\"Referr… │\n",
       "│ 1501615525 ┆ 20170801 ┆ Referral        ┆ {\"1501615525\",\"20170801\",\"Referr… │\n",
       "└────────────┴──────────┴─────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        'visitId',\n",
    "        'date',\n",
    "        'channelGrouping',\n",
    "        pl.struct('visitId', 'date', 'channelGrouping').alias('structFromCols')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (7, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channelGrouping</th><th>visitId</th><th>numVisits</th><th>struct_from_list</th></tr><tr><td>str</td><td>list[str]</td><td>u32</td><td>struct[12]</td></tr></thead><tbody><tr><td>&quot;Display&quot;</td><td>[&quot;1501651856&quot;, &quot;1501625928&quot;, … &quot;1501638116&quot;]</td><td>12</td><td>{&quot;1501651856&quot;,&quot;1501625928&quot;,&quot;1501611633&quot;,&quot;1501625068&quot;,&quot;1501612878&quot;,&quot;1501616158&quot;,&quot;1501607332&quot;,&quot;1501622703&quot;,&quot;1501621231&quot;,&quot;1501649570&quot;,&quot;1501647417&quot;,&quot;1501638116&quot;}</td></tr><tr><td>&quot;Paid Search&quot;</td><td>[&quot;1501610896&quot;, &quot;1501644116&quot;, … &quot;1501613648&quot;]</td><td>20</td><td>{&quot;1501610896&quot;,&quot;1501644116&quot;,&quot;1501574187&quot;,&quot;1501614708&quot;,&quot;1501625398&quot;,&quot;1501636143&quot;,&quot;1501617844&quot;,&quot;1501628199&quot;,&quot;1501618328&quot;,&quot;1501624893&quot;,&quot;1501650118&quot;,&quot;1501580936&quot;}</td></tr><tr><td>&quot;Affiliates&quot;</td><td>[&quot;1501604627&quot;, &quot;1501572101&quot;, … &quot;1501635918&quot;]</td><td>29</td><td>{&quot;1501604627&quot;,&quot;1501572101&quot;,&quot;1501638418&quot;,&quot;1501589595&quot;,&quot;1501635523&quot;,&quot;1501588687&quot;,&quot;1501602677&quot;,&quot;1501656633&quot;,&quot;1501589444&quot;,&quot;1501654572&quot;,&quot;1501588961&quot;,&quot;1501588902&quot;}</td></tr><tr><td>&quot;Referral&quot;</td><td>[&quot;1501589647&quot;, &quot;1501616621&quot;, … &quot;1501607798&quot;]</td><td>106</td><td>{&quot;1501589647&quot;,&quot;1501616621&quot;,&quot;1501601200&quot;,&quot;1501615525&quot;,&quot;1501589650&quot;,&quot;1501573710&quot;,&quot;1501613382&quot;,&quot;1501630140&quot;,&quot;1501656976&quot;,&quot;1501602227&quot;,&quot;1501620300&quot;,&quot;1501611288&quot;}</td></tr><tr><td>&quot;Social&quot;</td><td>[&quot;1501590147&quot;, &quot;1501655923&quot;, … &quot;1501652602&quot;]</td><td>136</td><td>{&quot;1501590147&quot;,&quot;1501655923&quot;,&quot;1501640054&quot;,&quot;1501596419&quot;,&quot;1501591307&quot;,&quot;1501616949&quot;,&quot;1501649584&quot;,&quot;1501579329&quot;,&quot;1501585058&quot;,&quot;1501618027&quot;,&quot;1501653304&quot;,&quot;1501614595&quot;}</td></tr><tr><td>&quot;Direct&quot;</td><td>[&quot;1501586309&quot;, &quot;1501587435&quot;, … &quot;1501610792&quot;]</td><td>163</td><td>{&quot;1501586309&quot;,&quot;1501587435&quot;,&quot;1501653660&quot;,&quot;1501608816&quot;,&quot;1501611913&quot;,&quot;1501584277&quot;,&quot;1501578373&quot;,&quot;1501587465&quot;,&quot;1501621325&quot;,&quot;1501655032&quot;,&quot;1501622827&quot;,&quot;1501575169&quot;}</td></tr><tr><td>&quot;Organic Search…</td><td>[&quot;1501591568&quot;, &quot;1501583103&quot;, … &quot;1501625964&quot;]</td><td>534</td><td>{&quot;1501591568&quot;,&quot;1501583103&quot;,&quot;1501631547&quot;,&quot;1501599064&quot;,&quot;1501585229&quot;,&quot;1501639903&quot;,&quot;1501576309&quot;,&quot;1501573981&quot;,&quot;1501618526&quot;,&quot;1501578968&quot;,&quot;1501599268&quot;,&quot;1501596177&quot;}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (7, 4)\n",
       "┌─────────────────┬──────────────────────────────────┬───────────┬─────────────────────────────────┐\n",
       "│ channelGrouping ┆ visitId                          ┆ numVisits ┆ struct_from_list                │\n",
       "│ ---             ┆ ---                              ┆ ---       ┆ ---                             │\n",
       "│ str             ┆ list[str]                        ┆ u32       ┆ struct[12]                      │\n",
       "╞═════════════════╪══════════════════════════════════╪═══════════╪═════════════════════════════════╡\n",
       "│ Display         ┆ [\"1501651856\", \"1501625928\", …   ┆ 12        ┆ {\"1501651856\",\"1501625928\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Paid Search     ┆ [\"1501610896\", \"1501644116\", …   ┆ 20        ┆ {\"1501610896\",\"1501644116\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Affiliates      ┆ [\"1501604627\", \"1501572101\", …   ┆ 29        ┆ {\"1501604627\",\"1501572101\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Referral        ┆ [\"1501589647\", \"1501616621\", …   ┆ 106       ┆ {\"1501589647\",\"1501616621\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Social          ┆ [\"1501590147\", \"1501655923\", …   ┆ 136       ┆ {\"1501590147\",\"1501655923\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Direct          ┆ [\"1501586309\", \"1501587435\", …   ┆ 163       ┆ {\"1501586309\",\"1501587435\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "│ Organic Search  ┆ [\"1501591568\", \"1501583103\", …   ┆ 534       ┆ {\"1501591568\",\"1501583103\",\"150 │\n",
       "│                 ┆ \"…                               ┆           ┆ 1…                              │\n",
       "└─────────────────┴──────────────────────────────────┴───────────┴─────────────────────────────────┘"
      ]
     },
     "execution_count": 450,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .group_by('channelGrouping')\n",
    "    .agg(\n",
    "        'visitId', \n",
    "        pl.col('visitId').len().alias('numVisits')\n",
    "    )\n",
    "    .sort('numVisits')\n",
    "    .with_columns(\n",
    "        pl.col('visitId').list.to_struct().alias('struct_from_list')\n",
    "    )   \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>structFromCols</th><th>visitId</th><th>date</th><th>channelGrouping</th></tr><tr><td>struct[3]</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>{&quot;1501591568&quot;,&quot;20170801&quot;,&quot;Organic Search&quot;}</td><td>&quot;1501591568&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Organic Search…</td></tr><tr><td>{&quot;1501589647&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501589647&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501616621&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501616621&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501601200&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501601200&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501615525&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501615525&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌───────────────────────────────────┬────────────┬──────────┬─────────────────┐\n",
       "│ structFromCols                    ┆ visitId    ┆ date     ┆ channelGrouping │\n",
       "│ ---                               ┆ ---        ┆ ---      ┆ ---             │\n",
       "│ struct[3]                         ┆ str        ┆ str      ┆ str             │\n",
       "╞═══════════════════════════════════╪════════════╪══════════╪═════════════════╡\n",
       "│ {\"1501591568\",\"20170801\",\"Organi… ┆ 1501591568 ┆ 20170801 ┆ Organic Search  │\n",
       "│ {\"1501589647\",\"20170801\",\"Referr… ┆ 1501589647 ┆ 20170801 ┆ Referral        │\n",
       "│ {\"1501616621\",\"20170801\",\"Referr… ┆ 1501616621 ┆ 20170801 ┆ Referral        │\n",
       "│ {\"1501601200\",\"20170801\",\"Referr… ┆ 1501601200 ┆ 20170801 ┆ Referral        │\n",
       "│ {\"1501615525\",\"20170801\",\"Referr… ┆ 1501615525 ┆ 20170801 ┆ Referral        │\n",
       "└───────────────────────────────────┴────────────┴──────────┴─────────────────┘"
      ]
     },
     "execution_count": 441,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        'structFromCols',\n",
    "        pl.col('structFromCols').alias('structFromColsToBeUnpacked')\n",
    "    )\n",
    "    .unnest('structFromColsToBeUnpacked')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>referralPath</th><th>campaign</th><th>source</th><th>medium</th><th>keyword</th><th>adContent</th><th>adwordsClickInfo</th><th>isTrueDirect</th><th>campaignCode</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>null</td><td>struct[12]</td><td>str</td><td>null</td></tr></thead><tbody><tr><td>null</td><td>&quot;(not set)&quot;</td><td>&quot;(direct)&quot;</td><td>&quot;(none)&quot;</td><td>null</td><td>null</td><td>{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null}</td><td>null</td><td>null</td></tr><tr><td>&quot;/analytics/web…</td><td>&quot;(not set)&quot;</td><td>&quot;analytics.goog…</td><td>&quot;referral&quot;</td><td>null</td><td>null</td><td>{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null}</td><td>null</td><td>null</td></tr><tr><td>&quot;/analytics/web…</td><td>&quot;(not set)&quot;</td><td>&quot;analytics.goog…</td><td>&quot;referral&quot;</td><td>null</td><td>null</td><td>{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null}</td><td>null</td><td>null</td></tr><tr><td>&quot;/analytics/web…</td><td>&quot;(not set)&quot;</td><td>&quot;analytics.goog…</td><td>&quot;referral&quot;</td><td>null</td><td>null</td><td>{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null}</td><td>null</td><td>null</td></tr><tr><td>&quot;/analytics/web…</td><td>&quot;(not set)&quot;</td><td>&quot;adwords.google…</td><td>&quot;referral&quot;</td><td>null</td><td>null</td><td>{null,null,null,null,null,null,&quot;not available in demo dataset&quot;,null,null,null,null,null}</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 9)\n",
       "┌───────────┬───────────┬───────────┬──────────┬───┬───────────┬───────────┬───────────┬───────────┐\n",
       "│ referralP ┆ campaign  ┆ source    ┆ medium   ┆ … ┆ adContent ┆ adwordsCl ┆ isTrueDir ┆ campaignC │\n",
       "│ ath       ┆ ---       ┆ ---       ┆ ---      ┆   ┆ ---       ┆ ickInfo   ┆ ect       ┆ ode       │\n",
       "│ ---       ┆ str       ┆ str       ┆ str      ┆   ┆ null      ┆ ---       ┆ ---       ┆ ---       │\n",
       "│ str       ┆           ┆           ┆          ┆   ┆           ┆ struct[12 ┆ str       ┆ null      │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ]         ┆           ┆           │\n",
       "╞═══════════╪═══════════╪═══════════╪══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n",
       "│ null      ┆ (not set) ┆ (direct)  ┆ (none)   ┆ … ┆ null      ┆ {null,nul ┆ null      ┆ null      │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ l,null,nu ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ll,null,n ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ull,\"…    ┆           ┆           │\n",
       "│ /analytic ┆ (not set) ┆ analytics ┆ referral ┆ … ┆ null      ┆ {null,nul ┆ null      ┆ null      │\n",
       "│ s/web/    ┆           ┆ .google.c ┆          ┆   ┆           ┆ l,null,nu ┆           ┆           │\n",
       "│           ┆           ┆ om        ┆          ┆   ┆           ┆ ll,null,n ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ull,\"…    ┆           ┆           │\n",
       "│ /analytic ┆ (not set) ┆ analytics ┆ referral ┆ … ┆ null      ┆ {null,nul ┆ null      ┆ null      │\n",
       "│ s/web/    ┆           ┆ .google.c ┆          ┆   ┆           ┆ l,null,nu ┆           ┆           │\n",
       "│           ┆           ┆ om        ┆          ┆   ┆           ┆ ll,null,n ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ull,\"…    ┆           ┆           │\n",
       "│ /analytic ┆ (not set) ┆ analytics ┆ referral ┆ … ┆ null      ┆ {null,nul ┆ null      ┆ null      │\n",
       "│ s/web/    ┆           ┆ .google.c ┆          ┆   ┆           ┆ l,null,nu ┆           ┆           │\n",
       "│           ┆           ┆ om        ┆          ┆   ┆           ┆ ll,null,n ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ull,\"…    ┆           ┆           │\n",
       "│ /analytic ┆ (not set) ┆ adwords.g ┆ referral ┆ … ┆ null      ┆ {null,nul ┆ null      ┆ null      │\n",
       "│ s/web/    ┆           ┆ oogle.com ┆          ┆   ┆           ┆ l,null,nu ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ll,null,n ┆           ┆           │\n",
       "│           ┆           ┆           ┆          ┆   ┆           ┆ ull,\"…    ┆           ┆           │\n",
       "└───────────┴───────────┴───────────┴──────────┴───┴───────────┴───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 509,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        pl.col('trafficSource')\n",
    "    )\n",
    "    .unnest('trafficSource')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>structFromCols</th><th>a</th><th>b</th><th>c</th></tr><tr><td>struct[3]</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>{&quot;1501591568&quot;,&quot;20170801&quot;,&quot;Organic Search&quot;}</td><td>&quot;1501591568&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Organic Search…</td></tr><tr><td>{&quot;1501589647&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501589647&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501616621&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501616621&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501601200&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501601200&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501615525&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;1501615525&quot;</td><td>&quot;20170801&quot;</td><td>&quot;Referral&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌───────────────────────────────────┬────────────┬──────────┬────────────────┐\n",
       "│ structFromCols                    ┆ a          ┆ b        ┆ c              │\n",
       "│ ---                               ┆ ---        ┆ ---      ┆ ---            │\n",
       "│ struct[3]                         ┆ str        ┆ str      ┆ str            │\n",
       "╞═══════════════════════════════════╪════════════╪══════════╪════════════════╡\n",
       "│ {\"1501591568\",\"20170801\",\"Organi… ┆ 1501591568 ┆ 20170801 ┆ Organic Search │\n",
       "│ {\"1501589647\",\"20170801\",\"Referr… ┆ 1501589647 ┆ 20170801 ┆ Referral       │\n",
       "│ {\"1501616621\",\"20170801\",\"Referr… ┆ 1501616621 ┆ 20170801 ┆ Referral       │\n",
       "│ {\"1501601200\",\"20170801\",\"Referr… ┆ 1501601200 ┆ 20170801 ┆ Referral       │\n",
       "│ {\"1501615525\",\"20170801\",\"Referr… ┆ 1501615525 ┆ 20170801 ┆ Referral       │\n",
       "└───────────────────────────────────┴────────────┴──────────┴────────────────┘"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        'structFromCols',\n",
    "        pl.col('structFromCols').struct.rename_fields(['a', 'b', 'c']).alias('renamedStructToBeUnpacked')\n",
    "    )\n",
    "    .unnest('renamedStructToBeUnpacked')\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 533,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>structFromCols</th><th>channelGrouping</th></tr><tr><td>struct[3]</td><td>str</td></tr></thead><tbody><tr><td>{&quot;1501591568&quot;,&quot;20170801&quot;,&quot;Organic Search&quot;}</td><td>&quot;Organic Search…</td></tr><tr><td>{&quot;1501589647&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501616621&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501601200&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;Referral&quot;</td></tr><tr><td>{&quot;1501615525&quot;,&quot;20170801&quot;,&quot;Referral&quot;}</td><td>&quot;Referral&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────────────────────────┬─────────────────┐\n",
       "│ structFromCols                    ┆ channelGrouping │\n",
       "│ ---                               ┆ ---             │\n",
       "│ struct[3]                         ┆ str             │\n",
       "╞═══════════════════════════════════╪═════════════════╡\n",
       "│ {\"1501591568\",\"20170801\",\"Organi… ┆ Organic Search  │\n",
       "│ {\"1501589647\",\"20170801\",\"Referr… ┆ Referral        │\n",
       "│ {\"1501616621\",\"20170801\",\"Referr… ┆ Referral        │\n",
       "│ {\"1501601200\",\"20170801\",\"Referr… ┆ Referral        │\n",
       "│ {\"1501615525\",\"20170801\",\"Referr… ┆ Referral        │\n",
       "└───────────────────────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 533,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        'structFromCols',\n",
    "        pl.col('structFromCols').struct.field('channelGrouping')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 529,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (28, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>channelGrouping</th><th>source</th></tr><tr><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;Affiliates&quot;</td><td>&quot;Partners&quot;</td></tr><tr><td>&quot;Direct&quot;</td><td>&quot;(direct)&quot;</td></tr><tr><td>&quot;Display&quot;</td><td>&quot;(direct)&quot;</td></tr><tr><td>&quot;Display&quot;</td><td>&quot;dfa&quot;</td></tr><tr><td>&quot;Organic Search…</td><td>&quot;(direct)&quot;</td></tr><tr><td>&quot;Organic Search…</td><td>&quot;ask&quot;</td></tr><tr><td>&quot;Organic Search…</td><td>&quot;baidu&quot;</td></tr><tr><td>&quot;Paid Search&quot;</td><td>&quot;(direct)&quot;</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;(direct)&quot;</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;adwords.google…</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;analytics.goog…</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;blog.golang.or…</td></tr><tr><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;ph.search.yaho…</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;productforums.…</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;qiita.com&quot;</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;sashihara.jp&quot;</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;sites.google.c…</td></tr><tr><td>&quot;Referral&quot;</td><td>&quot;support.google…</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;facebook.com&quot;</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;groups.google.…</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;l.facebook.com…</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;m.facebook.com…</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;quora.com&quot;</td></tr><tr><td>&quot;Social&quot;</td><td>&quot;youtube.com&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (28, 2)\n",
       "┌─────────────────┬────────────────┐\n",
       "│ channelGrouping ┆ source         │\n",
       "│ ---             ┆ ---            │\n",
       "│ str             ┆ str            │\n",
       "╞═════════════════╪════════════════╡\n",
       "│ Affiliates      ┆ Partners       │\n",
       "│ Direct          ┆ (direct)       │\n",
       "│ Display         ┆ (direct)       │\n",
       "│ Display         ┆ dfa            │\n",
       "│ …               ┆ …              │\n",
       "│ Social          ┆ l.facebook.com │\n",
       "│ Social          ┆ m.facebook.com │\n",
       "│ Social          ┆ quora.com      │\n",
       "│ Social          ┆ youtube.com    │\n",
       "└─────────────────┴────────────────┘"
      ]
     },
     "execution_count": 529,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    df\n",
    "    .select(\n",
    "        pl.struct(\n",
    "            pl.col('channelGrouping'),\n",
    "            pl.col('trafficSource').struct.field('source')\n",
    "        )\n",
    "        .unique()\n",
    "        .alias('channelAndSource')\n",
    "    )\n",
    "    .unnest('channelAndSource')\n",
    "    .sort('channelGrouping', 'source')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There is more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>total_str</th><th>total_struct</th></tr><tr><td>str</td><td>struct[13]</td></tr></thead><tbody><tr><td>&quot;{&quot;visits&quot;:&quot;1&quot;,…</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td></tr><tr><td>&quot;{&quot;visits&quot;:&quot;1&quot;,…</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,null,null,null,null,null,&quot;1&quot;}</td></tr><tr><td>&quot;{&quot;visits&quot;:&quot;1&quot;,…</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td></tr><tr><td>&quot;{&quot;visits&quot;:&quot;1&quot;,…</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td></tr><tr><td>&quot;{&quot;visits&quot;:&quot;1&quot;,…</td><td>{&quot;1&quot;,&quot;1&quot;,&quot;1&quot;,null,&quot;1&quot;,null,null,&quot;1&quot;,null,null,null,null,&quot;1&quot;}</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────────────────────────┬───────────────────────────────────┐\n",
       "│ total_str                         ┆ total_struct                      │\n",
       "│ ---                               ┆ ---                               │\n",
       "│ str                               ┆ struct[13]                        │\n",
       "╞═══════════════════════════════════╪═══════════════════════════════════╡\n",
       "│ {\"visits\":\"1\",\"hits\":\"1\",\"pagevi… ┆ {\"1\",\"1\",\"1\",null,\"1\",null,null,… │\n",
       "│ {\"visits\":\"1\",\"hits\":\"1\",\"pagevi… ┆ {\"1\",\"1\",\"1\",null,\"1\",null,null,… │\n",
       "│ {\"visits\":\"1\",\"hits\":\"1\",\"pagevi… ┆ {\"1\",\"1\",\"1\",null,\"1\",null,null,… │\n",
       "│ {\"visits\":\"1\",\"hits\":\"1\",\"pagevi… ┆ {\"1\",\"1\",\"1\",null,\"1\",null,null,… │\n",
       "│ {\"visits\":\"1\",\"hits\":\"1\",\"pagevi… ┆ {\"1\",\"1\",\"1\",null,\"1\",null,null,… │\n",
       "└───────────────────────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 545,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_struct_to_str_expr = pl.col('totals').struct.json_encode()\n",
    "(\n",
    "    df\n",
    "    .select(\n",
    "        total_struct_to_str_expr.alias('total_str'),\n",
    "        total_struct_to_str_expr\n",
    "        .str.json_decode()\n",
    "        .alias('total_struct')\n",
    "    )\n",
    ").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
